good_pipeline 0.1.0

data/lib/good_pipeline/pipeline.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module GoodPipeline
+  class Pipeline
+    VALID_FAILURE_STRATEGIES = %i[halt continue ignore].freeze
+    DSL_ATTRIBUTES = %i[description failure_strategy on_complete on_success on_failure].freeze
+
+    # --- Class-level DSL ---
+
+    class << self
+      def inherited(subclass)
+        super
+        DSL_ATTRIBUTES.each do |attribute|
+          subclass.instance_variable_set(:"@#{attribute}", instance_variable_get(:"@#{attribute}"))
+        end
+      end
+
+      def description(text = :__unset__)
+        return @description if text == :__unset__
+
+        @description = text
+      end
+
+      def failure_strategy(strategy = :__unset__)
+        return @failure_strategy || :halt if strategy == :__unset__
+
+        unless VALID_FAILURE_STRATEGIES.include?(strategy)
+          valid = VALID_FAILURE_STRATEGIES.map { |valid_strategy| ":#{valid_strategy}" }.join(", ")
+          raise ConfigurationError, "invalid failure strategy :#{strategy}, must be one of #{valid}"
+        end
+
+        @failure_strategy = strategy
+      end
+
+      def on_complete(method_name = :__unset__)
+        return @on_complete if method_name == :__unset__
+
+        @on_complete = method_name
+      end
+
+      def on_success(method_name = :__unset__)
+        return @on_success if method_name == :__unset__
+
+        @on_success = method_name
+      end
+
+      def on_failure(method_name = :__unset__)
+        return @on_failure if method_name == :__unset__
+
+        @on_failure = method_name
+      end
+
+      alias build new
+
+      def run(**)
+        instance = new(**)
+        pipeline_record = Runner.call(instance)
+        Chain.new(pipeline_record)
+      end
+
+      # Internal: reconstructs a minimal Pipeline for callback dispatch.
+      def for_callback(pipeline_record)
+        instance = allocate
+        instance.instance_variable_set(:@pipeline_record, pipeline_record)
+        instance.instance_variable_set(:@params, pipeline_record.params.symbolize_keys.freeze)
+        instance.instance_variable_set(:@step_definitions, [].freeze)
+        instance.instance_variable_set(:@steps_by_key, {}.freeze)
+        instance.instance_variable_set(:@root_steps, [].freeze)
+        instance
+      end
+    end
+
+    # --- Instance API ---
+
+    attr_reader :step_definitions, :steps_by_key, :root_steps, :params, :pipeline_record
+
+    def description
+      self.class.description
+    end
+
+    def failure_strategy
+      self.class.failure_strategy
+    end
+
+    def on_complete_callback = self.class.on_complete
+    def on_success_callback = self.class.on_success
+    def on_failure_callback = self.class.on_failure
+
+    def initialize(**kwargs)
+      @params = kwargs.freeze
+      @step_definitions = []
+      @building = true
+      configure(**kwargs)
+      GraphValidator.validate!(@step_definitions)
+      @step_definitions.freeze
+      @building = false
+      @steps_by_key = @step_definitions.to_h { |step| [step.key, step] }.freeze
+      @root_steps = @step_definitions.select { |step| step.dependencies.empty? }.freeze
+      freeze
+    end
+
+    private
+
+    def configure(**_kwargs)
+      raise NotImplementedError, "#{self.class} must implement #configure"
+    end
+
+    def run(key, job_class, with: {}, after: [], failure_strategy: nil, queue: nil, priority: nil)
+      raise ConfigurationError, "run can only be called inside configure" unless @building
+
+      @step_definitions << StepDefinition.new(
+        key: key,
+        job_class: job_class,
+        params: with,
+        dependencies: after,
+        failure_strategy: failure_strategy,
+        queue: queue,
+        priority: priority
+      )
+    end
+  end
+end

data/lib/good_pipeline/runner.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module GoodPipeline
+  class Runner
+    def self.call(pipeline_instance, start: true)
+      new(pipeline_instance).call(start: start)
+    end
+
+    def initialize(pipeline_instance)
+      @pipeline = pipeline_instance
+    end
+
+    def call(start: true) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+      pipeline_record = nil
+      step_records = {}
+
+      PipelineRecord.transaction do # rubocop:disable Metrics/BlockLength
+        pipeline_record = PipelineRecord.create!(
+          type: @pipeline.class.name,
+          params: @pipeline.params,
+          status: :pending,
+          on_failure_strategy: @pipeline.failure_strategy.to_s
+        )
+
+        @pipeline.step_definitions.each do |step_definition|
+          step_records[step_definition.key] = StepRecord.create!(
+            pipeline: pipeline_record,
+            key: step_definition.key.to_s,
+            job_class: step_definition.job_class.name,
+            params: step_definition.params,
+            on_failure_strategy: step_definition.failure_strategy&.to_s,
+            queue: step_definition.queue,
+            priority: step_definition.priority
+          )
+
+          step_definition.dependencies.each do |dependency_key|
+            DependencyRecord.create!(
+              pipeline: pipeline_record,
+              step: step_records[step_definition.key],
+              depends_on_step: step_records[dependency_key]
+            )
+          end
+        end
+
+        pipeline_batch = GoodJob::Batch.new
+        pipeline_batch.on_finish = "GoodPipeline::PipelineReconciliationJob"
+        pipeline_batch.properties = { pipeline_id: pipeline_record.id }
+        pipeline_batch.save
+        pipeline_record.update_column(:good_job_batch_id, pipeline_batch.id)
+
+        pipeline_record.transition_to!(:running) if start
+      end
+
+      if start
+        @pipeline.root_steps.each do |step_definition|
+          Coordinator.try_enqueue_step(step_records[step_definition.key].id)
+        end
+      end
+
+      pipeline_record
+    end
+  end
+end

data/lib/good_pipeline/step_definition.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module GoodPipeline
+  class StepDefinition
+    attr_reader :key, :job_class, :params, :dependencies, :failure_strategy, :queue, :priority
+
+    def initialize(key:, job_class:, params: {}, dependencies: [], failure_strategy: nil, queue: nil, priority: nil)
+      @key = key
+      @job_class = job_class
+      @params = params.freeze
+      @dependencies = Array(dependencies).freeze
+      @failure_strategy = failure_strategy
+      @queue = queue
+      @priority = priority
+      freeze
+    end
+  end
+end

data/lib/good_pipeline/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module GoodPipeline
+  VERSION = "0.1.0"
+end

data/lib/good_pipeline.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require_relative "good_pipeline/version"
+require_relative "good_pipeline/errors"
+require_relative "good_pipeline/step_definition"
+require_relative "good_pipeline/cycle_detector"
+require_relative "good_pipeline/graph_validator"
+require_relative "good_pipeline/pipeline"
+require_relative "good_pipeline/failure_metadata"
+require_relative "good_pipeline/coordinator"
+require_relative "good_pipeline/chain_coordinator"
+require_relative "good_pipeline/runner"
+require_relative "good_pipeline/chain"
+require_relative "good_pipeline/engine" if defined?(Rails::Engine)
+
+module GoodPipeline
+  def self.run(*pipeline_configs)
+    pipeline_records = pipeline_configs.map do |config|
+      pipeline_class, pipeline_params = extract_pipeline_config(config)
+      instance = pipeline_class.build(**pipeline_params)
+      Runner.call(instance)
+    end
+
+    Chain.new(pipeline_records)
+  end
+
+  # Internal: parses [PipelineClass, { with: { ... } }] config format.
+  def self.extract_pipeline_config(config)
+    [config[0], config.fetch(1, {}).fetch(:with, {})]
+  end
+
+  def self.cleanup_preserved_pipelines(older_than:)
+    pipeline_ids = PipelineRecord.where(status: PipelineRecord::TERMINAL_STATUSES)
+                                 .where("updated_at < ?", older_than)
+                                 .pluck(:id)
+    return if pipeline_ids.empty?
+
+    DependencyRecord.where(pipeline_id: pipeline_ids).delete_all
+    StepRecord.where(pipeline_id: pipeline_ids).delete_all
+    ChainRecord.where(upstream_pipeline_id: pipeline_ids)
+               .or(ChainRecord.where(downstream_pipeline_id: pipeline_ids))
+               .delete_all
+    PipelineRecord.where(id: pipeline_ids).delete_all
+  end
+end

data/mise.toml

@@ -0,0 +1,10 @@
+[tools]
+ruby = "3.4.7"
+
+[tasks."docker:start"]
+description = "Start Docker services using docker-compose"
+run = "docker compose -f dev-docker-compose.yml up -d"
+
+[tasks."docker:stop"]
+description = "Stop Docker services using docker-compose"
+run = "docker compose -f dev-docker-compose.yml down"

data/sig/good_pipeline.rbs

@@ -0,0 +1,4 @@
+module GoodPipeline
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,204 @@
+--- !ruby/object:Gem::Specification
+name: good_pipeline
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Ali Hamdi Ali Fadel
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.1'
+- !ruby/object:Gem::Dependency
+  name: good_job
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.10'
+- !ruby/object:Gem::Dependency
+  name: railties
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.1'
+description: Define multi-step workflows as directed acyclic graphs where each step
+  is a GoodJob job. Handles dependency resolution, parallel execution, failure strategies,
+  and lifecycle callbacks.
+email:
+- aliosm1997@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- app/controllers/good_pipeline/application_controller.rb
+- app/controllers/good_pipeline/frontends_controller.rb
+- app/controllers/good_pipeline/pipelines_controller.rb
+- app/frontend/good_pipeline/style.css
+- app/helpers/good_pipeline/pipelines_helper.rb
+- app/jobs/good_pipeline/pipeline_callback_job.rb
+- app/jobs/good_pipeline/pipeline_reconciliation_job.rb
+- app/jobs/good_pipeline/step_finished_job.rb
+- app/models/good_pipeline/chain_record.rb
+- app/models/good_pipeline/dependency_record.rb
+- app/models/good_pipeline/pipeline_record.rb
+- app/models/good_pipeline/step_record.rb
+- app/views/good_pipeline/pipelines/_chain_links.html.erb
+- app/views/good_pipeline/pipelines/_pagination.html.erb
+- app/views/good_pipeline/pipelines/_pipeline_row.html.erb
+- app/views/good_pipeline/pipelines/_steps_table.html.erb
+- app/views/good_pipeline/pipelines/definitions.html.erb
+- app/views/good_pipeline/pipelines/index.html.erb
+- app/views/good_pipeline/pipelines/show.html.erb
+- app/views/layouts/good_pipeline/application.html.erb
+- config/routes.rb
+- demo/Rakefile
+- demo/app/jobs/always_failing_job.rb
+- demo/app/jobs/application_job.rb
+- demo/app/jobs/cleanup_job.rb
+- demo/app/jobs/download_job.rb
+- demo/app/jobs/failing_job.rb
+- demo/app/jobs/publish_job.rb
+- demo/app/jobs/retryable_job.rb
+- demo/app/jobs/thumbnail_job.rb
+- demo/app/jobs/transcode_job.rb
+- demo/app/pipelines/analytics_pipeline.rb
+- demo/app/pipelines/archive_pipeline.rb
+- demo/app/pipelines/continue_test_pipeline.rb
+- demo/app/pipelines/halt_test_pipeline.rb
+- demo/app/pipelines/notification_pipeline.rb
+- demo/app/pipelines/test_pipeline.rb
+- demo/app/pipelines/video_processing_pipeline.rb
+- demo/bin/rails
+- demo/config.ru
+- demo/config/application.rb
+- demo/config/boot.rb
+- demo/config/database.yml
+- demo/config/environment.rb
+- demo/config/environments/development.rb
+- demo/config/environments/test.rb
+- demo/config/routes.rb
+- demo/db/migrate/20260319205325_create_good_jobs.rb
+- demo/db/migrate/20260319205326_create_good_pipeline_tables.rb
+- demo/db/seeds.rb
+- demo/test/good_pipeline/test_chain_record.rb
+- demo/test/good_pipeline/test_cleanup.rb
+- demo/test/good_pipeline/test_coordinator.rb
+- demo/test/good_pipeline/test_dependency_record.rb
+- demo/test/good_pipeline/test_failure_metadata.rb
+- demo/test/good_pipeline/test_introspection.rb
+- demo/test/good_pipeline/test_pipeline_callback_job.rb
+- demo/test/good_pipeline/test_pipeline_reconciliation_job.rb
+- demo/test/good_pipeline/test_pipeline_record.rb
+- demo/test/good_pipeline/test_runner.rb
+- demo/test/good_pipeline/test_step_finished_job.rb
+- demo/test/good_pipeline/test_step_record.rb
+- demo/test/integration/test_concurrent_fan_in.rb
+- demo/test/integration/test_end_to_end.rb
+- demo/test/integration/test_enqueue_atomicity.rb
+- demo/test/integration/test_pipeline_chaining.rb
+- demo/test/integration/test_retry_scenarios.rb
+- demo/test/integration/test_step_finished_idempotency.rb
+- demo/test/test_helper.rb
+- dev-docker-compose.yml
+- docs/.vitepress/config.mts
+- docs/.vitepress/theme/custom.css
+- docs/.vitepress/theme/index.ts
+- docs/architecture.md
+- docs/callbacks.md
+- docs/cleanup.md
+- docs/dag-validation.md
+- docs/dashboard.md
+- docs/defining-pipelines.md
+- docs/failure-strategies.md
+- docs/getting-started.md
+- docs/index.md
+- docs/introduction.md
+- docs/monitoring.md
+- docs/package-lock.json
+- docs/package.json
+- docs/pipeline-chaining.md
+- docs/public/screenshots/definitions.png
+- docs/public/screenshots/index.png
+- docs/public/screenshots/show.png
+- docs/screenshots/definitions.png
+- docs/screenshots/index.png
+- docs/screenshots/show.png
+- lib/generators/good_pipeline/install/install_generator.rb
+- lib/generators/good_pipeline/install/templates/create_good_pipeline_tables.rb.erb
+- lib/good_pipeline.rb
+- lib/good_pipeline/chain.rb
+- lib/good_pipeline/chain_coordinator.rb
+- lib/good_pipeline/coordinator.rb
+- lib/good_pipeline/cycle_detector.rb
+- lib/good_pipeline/engine.rb
+- lib/good_pipeline/errors.rb
+- lib/good_pipeline/failure_metadata.rb
+- lib/good_pipeline/graph_validator.rb
+- lib/good_pipeline/pipeline.rb
+- lib/good_pipeline/runner.rb
+- lib/good_pipeline/step_definition.rb
+- lib/good_pipeline/version.rb
+- mise.toml
+- sig/good_pipeline.rbs
+homepage: https://github.com/milkstrawai/good_pipeline
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/milkstrawai/good_pipeline
+  source_code_uri: https://github.com/milkstrawai/good_pipeline
+  changelog_uri: https://github.com/milkstrawai/good_pipeline/blob/main/CHANGELOG.md
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: DAG-based job pipeline orchestration for Rails, built on GoodJob
+test_files: []