ductwork 0.26.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. checksums.yaml +4 -4
  2. data/.claude/skills/audit-clock-drift/SKILL.md +11 -0
  3. data/.claude/skills/audit-database-indexes/SKILL.md +11 -0
  4. data/.claude/skills/audit-database-support/SKILL.md +18 -0
  5. data/.claude/skills/audit-durability/SKILL.md +17 -0
  6. data/.saturnci/database.yml +1 -0
  7. data/CHANGELOG-PRO.md +60 -0
  8. data/CHANGELOG.md +144 -1
  9. data/CLAUDE.md +34 -0
  10. data/README.md +24 -14
  11. data/app/controllers/ductwork/application_controller.rb +3 -3
  12. data/app/controllers/ductwork/dashboards_controller.rb +1 -1
  13. data/app/controllers/ductwork/pipelines_controller.rb +4 -3
  14. data/app/helpers/ductwork/application_helper.rb +2 -2
  15. data/app/views/ductwork/dashboards/show.html.erb +12 -12
  16. data/app/views/ductwork/pipelines/index.html.erb +12 -12
  17. data/app/views/ductwork/pipelines/show.html.erb +13 -13
  18. data/app/views/ductwork/step_errors/index.html.erb +2 -2
  19. data/lib/ductwork/branch_claim.rb +185 -0
  20. data/lib/ductwork/cli.rb +73 -24
  21. data/lib/ductwork/configuration.rb +73 -3
  22. data/lib/ductwork/context.rb +17 -19
  23. data/lib/ductwork/database_clock.rb +84 -0
  24. data/lib/ductwork/dsl/branch_builder.rb +6 -1
  25. data/lib/ductwork/dsl/definition_builder.rb +24 -1
  26. data/lib/ductwork/{job_claim.rb → execution_claim.rb} +7 -6
  27. data/lib/ductwork/fault_injection.rb +35 -0
  28. data/lib/ductwork/migration_helper.rb +1 -1
  29. data/lib/ductwork/models/advancement.rb +48 -0
  30. data/lib/ductwork/models/attempt.rb +9 -0
  31. data/lib/ductwork/models/availability.rb +2 -0
  32. data/lib/ductwork/models/branch.rb +800 -0
  33. data/lib/ductwork/models/branch_link.rb +10 -0
  34. data/lib/ductwork/models/execution.rb +199 -2
  35. data/lib/ductwork/models/job.rb +8 -121
  36. data/lib/ductwork/models/pipeline.rb +150 -272
  37. data/lib/ductwork/models/process.rb +177 -5
  38. data/lib/ductwork/models/result.rb +2 -1
  39. data/lib/ductwork/models/run.rb +119 -1
  40. data/lib/ductwork/models/step.rb +26 -6
  41. data/lib/ductwork/models/transition.rb +15 -0
  42. data/lib/ductwork/models/tuple.rb +1 -4
  43. data/lib/ductwork/models/workflow.rb +3 -0
  44. data/lib/ductwork/optimistic_locking_execution_claim.rb +97 -0
  45. data/lib/ductwork/processes/health_check.rb +87 -0
  46. data/lib/ductwork/processes/job_worker.rb +86 -20
  47. data/lib/ductwork/processes/job_worker_runner.rb +25 -37
  48. data/lib/ductwork/processes/pipeline_advancer.rb +48 -76
  49. data/lib/ductwork/processes/pipeline_advancer_runner.rb +34 -44
  50. data/lib/ductwork/processes/process_supervisor.rb +87 -10
  51. data/lib/ductwork/processes/thread_supervisor.rb +55 -21
  52. data/lib/ductwork/processes/thread_supervisor_runner.rb +8 -5
  53. data/lib/ductwork/processes/worker_health_check.rb +55 -0
  54. data/lib/ductwork/row_locking_execution_claim.rb +77 -0
  55. data/lib/ductwork/testing/rspec.rb +28 -11
  56. data/lib/ductwork/version.rb +1 -1
  57. data/lib/ductwork.rb +5 -0
  58. data/lib/generators/ductwork/install/install_generator.rb +14 -4
  59. data/lib/generators/ductwork/install/templates/config/ductwork.yml +4 -0
  60. data/lib/generators/ductwork/install/templates/db/create_ductwork_advancements.rb +29 -0
  61. data/lib/generators/ductwork/install/templates/db/create_ductwork_attempts.rb +20 -0
  62. data/lib/generators/ductwork/install/templates/db/create_ductwork_availabilities.rb +18 -5
  63. data/lib/generators/ductwork/install/templates/db/create_ductwork_branch_links.rb +27 -0
  64. data/lib/generators/ductwork/install/templates/db/create_ductwork_branches.rb +37 -0
  65. data/lib/generators/ductwork/install/templates/db/create_ductwork_executions.rb +8 -1
  66. data/lib/generators/ductwork/install/templates/db/create_ductwork_pipelines.rb +0 -8
  67. data/lib/generators/ductwork/install/templates/db/create_ductwork_processes.rb +2 -0
  68. data/lib/generators/ductwork/install/templates/db/create_ductwork_results.rb +1 -0
  69. data/lib/generators/ductwork/install/templates/db/create_ductwork_runs.rb +45 -6
  70. data/lib/generators/ductwork/install/templates/db/create_ductwork_steps.rb +20 -5
  71. data/lib/generators/ductwork/install/templates/db/create_ductwork_transitions.rb +43 -0
  72. data/lib/generators/ductwork/install/templates/db/create_ductwork_tuples.rb +3 -3
  73. data/lib/generators/ductwork/update/templates/db/add_crash_count_to_ductwork_advancements.rb +7 -0
  74. data/lib/generators/ductwork/update/templates/db/add_crash_count_to_ductwork_executions.rb +14 -0
  75. data/lib/generators/ductwork/update/templates/db/add_indexes_to_ductwork_results.rb +7 -0
  76. data/lib/generators/ductwork/update/templates/db/add_indexes_to_ductwork_runs.rb +9 -0
  77. data/lib/generators/ductwork/update/templates/db/add_indexes_to_ductwork_transitions.rb +16 -0
  78. data/lib/generators/ductwork/update/templates/db/add_pipeline_started_index_to_ductwork_runs.rb +7 -0
  79. data/lib/generators/ductwork/update/templates/db/add_process_id_to_ductwork_executions.rb +35 -0
  80. data/lib/generators/ductwork/update/templates/db/add_role_to_ductwork_processes.rb +20 -0
  81. data/lib/generators/ductwork/update/templates/db/associate_branches_to_runs.rb +74 -0
  82. data/lib/generators/ductwork/update/templates/db/associate_steps_to_branches.rb +24 -0
  83. data/lib/generators/ductwork/update/templates/db/associate_steps_to_runs.rb +72 -0
  84. data/lib/generators/ductwork/update/templates/db/associate_tuples_to_runs.rb +73 -0
  85. data/lib/generators/ductwork/update/templates/db/backfill_branch_ids_on_steps.rb +21 -0
  86. data/lib/generators/ductwork/update/templates/db/create_ductwork_advancements.rb +28 -0
  87. data/lib/generators/ductwork/update/templates/db/create_ductwork_branch_links.rb +27 -0
  88. data/lib/generators/ductwork/update/templates/db/create_ductwork_branches.rb +37 -0
  89. data/lib/generators/ductwork/update/templates/db/create_ductwork_runs.rb +55 -0
  90. data/lib/generators/ductwork/update/templates/db/create_ductwork_transitions.rb +32 -0
  91. data/lib/generators/ductwork/update/templates/db/denormalize_pipeline_klass_on_availabilities.rb +11 -4
  92. data/lib/generators/ductwork/update/templates/db/migrate_tables_to_uuid_primary_key.rb +5 -0
  93. data/lib/generators/ductwork/update/templates/db/rename_runs_to_attempts.rb +7 -0
  94. data/lib/generators/ductwork/update/templates/db/update_process_associations.rb +29 -0
  95. data/lib/generators/ductwork/update/update_generator.rb +99 -0
  96. metadata +47 -6
  97. data/lib/ductwork/optimistic_locking_job_claim.rb +0 -88
  98. data/lib/ductwork/row_locking_job_claim.rb +0 -75
@@ -1,75 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Ductwork
4
- class RowLockingJobClaim
5
- def initialize(klass)
6
- @id = nil
7
- @job = nil
8
- @klass = klass
9
- @process_id = ::Process.pid
10
- end
11
-
12
- def latest
13
- rows_updated = attempt_availability_claim
14
-
15
- if rows_updated == 1
16
- Ductwork.logger.debug(
17
- msg: "Job claimed",
18
- role: :job_worker,
19
- process_id: process_id,
20
- availability_id: id
21
- )
22
- update_state
23
- else
24
- Ductwork.logger.debug(
25
- msg: "No available job to claim",
26
- role: :job_worker,
27
- process_id: process_id,
28
- pipeline: klass
29
- )
30
- end
31
-
32
- job
33
- end
34
-
35
- private
36
-
37
- attr_reader :id, :job, :klass, :process_id
38
-
39
- def attempt_availability_claim
40
- Ductwork::Record.transaction do
41
- @id = Ductwork::Availability
42
- .where("ductwork_availabilities.started_at <= ?", Time.current)
43
- .where(completed_at: nil, pipeline_klass: klass)
44
- .order(:started_at)
45
- .lock("FOR UPDATE SKIP LOCKED")
46
- .limit(1)
47
- .ids
48
- .first
49
-
50
- if id.present?
51
- Ductwork::Availability
52
- .where(id: id, completed_at: nil)
53
- .update_all(completed_at: Time.current, process_id: process_id)
54
- else
55
- 0
56
- end
57
- end
58
- end
59
-
60
- def update_state
61
- Ductwork::Record.transaction do
62
- execution = Ductwork::Execution
63
- .joins(:availability)
64
- .where(completed_at: nil)
65
- .where(ductwork_availabilities: { id: })
66
- .sole
67
- @job = execution.job
68
-
69
- execution.update!(process_id:)
70
- job.step.in_progress!
71
- job.step.pipeline.in_progress!
72
- end
73
- end
74
- end
75
- end