ductwork 0.26.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.claude/skills/audit-clock-drift/SKILL.md +11 -0
- data/.claude/skills/audit-database-indexes/SKILL.md +11 -0
- data/.claude/skills/audit-database-support/SKILL.md +18 -0
- data/.claude/skills/audit-durability/SKILL.md +17 -0
- data/.saturnci/database.yml +1 -0
- data/CHANGELOG-PRO.md +60 -0
- data/CHANGELOG.md +144 -1
- data/CLAUDE.md +34 -0
- data/README.md +24 -14
- data/app/controllers/ductwork/application_controller.rb +3 -3
- data/app/controllers/ductwork/dashboards_controller.rb +1 -1
- data/app/controllers/ductwork/pipelines_controller.rb +4 -3
- data/app/helpers/ductwork/application_helper.rb +2 -2
- data/app/views/ductwork/dashboards/show.html.erb +12 -12
- data/app/views/ductwork/pipelines/index.html.erb +12 -12
- data/app/views/ductwork/pipelines/show.html.erb +13 -13
- data/app/views/ductwork/step_errors/index.html.erb +2 -2
- data/lib/ductwork/branch_claim.rb +185 -0
- data/lib/ductwork/cli.rb +73 -24
- data/lib/ductwork/configuration.rb +73 -3
- data/lib/ductwork/context.rb +17 -19
- data/lib/ductwork/database_clock.rb +84 -0
- data/lib/ductwork/dsl/branch_builder.rb +6 -1
- data/lib/ductwork/dsl/definition_builder.rb +24 -1
- data/lib/ductwork/{job_claim.rb → execution_claim.rb} +7 -6
- data/lib/ductwork/fault_injection.rb +35 -0
- data/lib/ductwork/migration_helper.rb +1 -1
- data/lib/ductwork/models/advancement.rb +48 -0
- data/lib/ductwork/models/attempt.rb +9 -0
- data/lib/ductwork/models/availability.rb +2 -0
- data/lib/ductwork/models/branch.rb +800 -0
- data/lib/ductwork/models/branch_link.rb +10 -0
- data/lib/ductwork/models/execution.rb +199 -2
- data/lib/ductwork/models/job.rb +8 -121
- data/lib/ductwork/models/pipeline.rb +150 -272
- data/lib/ductwork/models/process.rb +177 -5
- data/lib/ductwork/models/result.rb +2 -1
- data/lib/ductwork/models/run.rb +119 -1
- data/lib/ductwork/models/step.rb +26 -6
- data/lib/ductwork/models/transition.rb +15 -0
- data/lib/ductwork/models/tuple.rb +1 -4
- data/lib/ductwork/models/workflow.rb +3 -0
- data/lib/ductwork/optimistic_locking_execution_claim.rb +97 -0
- data/lib/ductwork/processes/health_check.rb +87 -0
- data/lib/ductwork/processes/job_worker.rb +86 -20
- data/lib/ductwork/processes/job_worker_runner.rb +25 -37
- data/lib/ductwork/processes/pipeline_advancer.rb +48 -76
- data/lib/ductwork/processes/pipeline_advancer_runner.rb +34 -44
- data/lib/ductwork/processes/process_supervisor.rb +87 -10
- data/lib/ductwork/processes/thread_supervisor.rb +55 -21
- data/lib/ductwork/processes/thread_supervisor_runner.rb +8 -5
- data/lib/ductwork/processes/worker_health_check.rb +55 -0
- data/lib/ductwork/row_locking_execution_claim.rb +77 -0
- data/lib/ductwork/testing/rspec.rb +28 -11
- data/lib/ductwork/version.rb +1 -1
- data/lib/ductwork.rb +5 -0
- data/lib/generators/ductwork/install/install_generator.rb +14 -4
- data/lib/generators/ductwork/install/templates/config/ductwork.yml +4 -0
- data/lib/generators/ductwork/install/templates/db/create_ductwork_advancements.rb +29 -0
- data/lib/generators/ductwork/install/templates/db/create_ductwork_attempts.rb +20 -0
- data/lib/generators/ductwork/install/templates/db/create_ductwork_availabilities.rb +18 -5
- data/lib/generators/ductwork/install/templates/db/create_ductwork_branch_links.rb +27 -0
- data/lib/generators/ductwork/install/templates/db/create_ductwork_branches.rb +37 -0
- data/lib/generators/ductwork/install/templates/db/create_ductwork_executions.rb +8 -1
- data/lib/generators/ductwork/install/templates/db/create_ductwork_pipelines.rb +0 -8
- data/lib/generators/ductwork/install/templates/db/create_ductwork_processes.rb +2 -0
- data/lib/generators/ductwork/install/templates/db/create_ductwork_results.rb +1 -0
- data/lib/generators/ductwork/install/templates/db/create_ductwork_runs.rb +45 -6
- data/lib/generators/ductwork/install/templates/db/create_ductwork_steps.rb +20 -5
- data/lib/generators/ductwork/install/templates/db/create_ductwork_transitions.rb +43 -0
- data/lib/generators/ductwork/install/templates/db/create_ductwork_tuples.rb +3 -3
- data/lib/generators/ductwork/update/templates/db/add_crash_count_to_ductwork_advancements.rb +7 -0
- data/lib/generators/ductwork/update/templates/db/add_crash_count_to_ductwork_executions.rb +14 -0
- data/lib/generators/ductwork/update/templates/db/add_indexes_to_ductwork_results.rb +7 -0
- data/lib/generators/ductwork/update/templates/db/add_indexes_to_ductwork_runs.rb +9 -0
- data/lib/generators/ductwork/update/templates/db/add_indexes_to_ductwork_transitions.rb +16 -0
- data/lib/generators/ductwork/update/templates/db/add_pipeline_started_index_to_ductwork_runs.rb +7 -0
- data/lib/generators/ductwork/update/templates/db/add_process_id_to_ductwork_executions.rb +35 -0
- data/lib/generators/ductwork/update/templates/db/add_role_to_ductwork_processes.rb +20 -0
- data/lib/generators/ductwork/update/templates/db/associate_branches_to_runs.rb +74 -0
- data/lib/generators/ductwork/update/templates/db/associate_steps_to_branches.rb +24 -0
- data/lib/generators/ductwork/update/templates/db/associate_steps_to_runs.rb +72 -0
- data/lib/generators/ductwork/update/templates/db/associate_tuples_to_runs.rb +73 -0
- data/lib/generators/ductwork/update/templates/db/backfill_branch_ids_on_steps.rb +21 -0
- data/lib/generators/ductwork/update/templates/db/create_ductwork_advancements.rb +28 -0
- data/lib/generators/ductwork/update/templates/db/create_ductwork_branch_links.rb +27 -0
- data/lib/generators/ductwork/update/templates/db/create_ductwork_branches.rb +37 -0
- data/lib/generators/ductwork/update/templates/db/create_ductwork_runs.rb +55 -0
- data/lib/generators/ductwork/update/templates/db/create_ductwork_transitions.rb +32 -0
- data/lib/generators/ductwork/update/templates/db/denormalize_pipeline_klass_on_availabilities.rb +11 -4
- data/lib/generators/ductwork/update/templates/db/migrate_tables_to_uuid_primary_key.rb +5 -0
- data/lib/generators/ductwork/update/templates/db/rename_runs_to_attempts.rb +7 -0
- data/lib/generators/ductwork/update/templates/db/update_process_associations.rb +29 -0
- data/lib/generators/ductwork/update/update_generator.rb +99 -0
- metadata +47 -6
- data/lib/ductwork/optimistic_locking_job_claim.rb +0 -88
- data/lib/ductwork/row_locking_job_claim.rb +0 -75
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Ductwork
|
|
4
|
-
class RowLockingJobClaim
|
|
5
|
-
def initialize(klass)
|
|
6
|
-
@id = nil
|
|
7
|
-
@job = nil
|
|
8
|
-
@klass = klass
|
|
9
|
-
@process_id = ::Process.pid
|
|
10
|
-
end
|
|
11
|
-
|
|
12
|
-
def latest
|
|
13
|
-
rows_updated = attempt_availability_claim
|
|
14
|
-
|
|
15
|
-
if rows_updated == 1
|
|
16
|
-
Ductwork.logger.debug(
|
|
17
|
-
msg: "Job claimed",
|
|
18
|
-
role: :job_worker,
|
|
19
|
-
process_id: process_id,
|
|
20
|
-
availability_id: id
|
|
21
|
-
)
|
|
22
|
-
update_state
|
|
23
|
-
else
|
|
24
|
-
Ductwork.logger.debug(
|
|
25
|
-
msg: "No available job to claim",
|
|
26
|
-
role: :job_worker,
|
|
27
|
-
process_id: process_id,
|
|
28
|
-
pipeline: klass
|
|
29
|
-
)
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
job
|
|
33
|
-
end
|
|
34
|
-
|
|
35
|
-
private
|
|
36
|
-
|
|
37
|
-
attr_reader :id, :job, :klass, :process_id
|
|
38
|
-
|
|
39
|
-
def attempt_availability_claim
|
|
40
|
-
Ductwork::Record.transaction do
|
|
41
|
-
@id = Ductwork::Availability
|
|
42
|
-
.where("ductwork_availabilities.started_at <= ?", Time.current)
|
|
43
|
-
.where(completed_at: nil, pipeline_klass: klass)
|
|
44
|
-
.order(:started_at)
|
|
45
|
-
.lock("FOR UPDATE SKIP LOCKED")
|
|
46
|
-
.limit(1)
|
|
47
|
-
.ids
|
|
48
|
-
.first
|
|
49
|
-
|
|
50
|
-
if id.present?
|
|
51
|
-
Ductwork::Availability
|
|
52
|
-
.where(id: id, completed_at: nil)
|
|
53
|
-
.update_all(completed_at: Time.current, process_id: process_id)
|
|
54
|
-
else
|
|
55
|
-
0
|
|
56
|
-
end
|
|
57
|
-
end
|
|
58
|
-
end
|
|
59
|
-
|
|
60
|
-
def update_state
|
|
61
|
-
Ductwork::Record.transaction do
|
|
62
|
-
execution = Ductwork::Execution
|
|
63
|
-
.joins(:availability)
|
|
64
|
-
.where(completed_at: nil)
|
|
65
|
-
.where(ductwork_availabilities: { id: })
|
|
66
|
-
.sole
|
|
67
|
-
@job = execution.job
|
|
68
|
-
|
|
69
|
-
execution.update!(process_id:)
|
|
70
|
-
job.step.in_progress!
|
|
71
|
-
job.step.pipeline.in_progress!
|
|
72
|
-
end
|
|
73
|
-
end
|
|
74
|
-
end
|
|
75
|
-
end
|