sfn_job 0.1.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a82f27e9eb0a157217986266bcc79a6dbce08ca4a2729315bf4a6e743408daf8
+  data.tar.gz: 7273359c3d3dee2e88e55bf7c13a1559eb1be81cac9ec9e0b8b147cfe12dfc0b
+SHA512:
+  metadata.gz: 8aa931901a48dd5d2ef0e8dcd98df1c80513f69d9f496b6599b8371e6b6a819ae8115bdef48473edc5c0845e3b3c565e961a355093deab7b827808fef1e7f97e
+  data.tar.gz: '014449fc5d255a07c5957df6aba749337b7eeb0cff117bbe57fc21cb4c6875249598d2f2fb411aa9382ef0443ff7168fc28d24875aad7b7b55eb298aa57687cc'

data/README.md

@@ -0,0 +1,75 @@
+# SfnJob
+
+SfnJob provides a simple way to run a job on AWS Step Functions.
+
+This might be useful when your team use sidekiq on container
+which makes it hard to run a job takes long time.
+
+## Installation
+
+```
+gem install sfn_job
+# or bundle add sfn_job
+```
+
+## Configuration
+
+```ruby
+SfnJob.configure do |config|
+  config.region = "ap-northeast-1"
+  config.account_id = "123456789012"
+  config.stub_sfn_client = Rails.env.test? # Set this to true when test not to call AWS API actually
+end
+```
+
+## Usage
+
+this gem provides active job adapter and runner task for enqueued job
+
+```ruby
+class SomeJob < ApplicationJob
+  self.queue_adapter = :sfn_job
+  # Treat state machine name as queue name
+  queue_as "sfn_job"
+
+  def perform(item_id)
+    # do something
+  end
+end
+```
+
+The gem will call StartExecution API with input as below:
+
+```json
+{
+  "serialized_job": "serialized_job_as_json"
+}
+```
+
+And create state machine which pass this serialized_job to RunTask overrided env "SERIEALIZED_JOB".
+
+```bash
+SERIEALIZED_JOB='serialized_job_as_json' bundle exec rails sfn_job:execute
+```
+
+## Why
+
+It's quite complex to handle long time async job,
+such as [sidekiq-iteration](https://github.com/fatkodima/sidekiq-iteration) or split job into small pieces with `Sideiq::Batch`.
+
+## Why not
+
+- hard to accept more infra complexity
+- which jobs are enqueued frequently
+  - Be careful with sfn StartExecution api rate limit. [Step Functions service quotas - AWS](https://docs.aws.amazon.com/step-functions/latest/dg/service-quotas.html)
+  - And RunTask api rate limit also (if you use Fargate) [AWS Fargate throttling quotas - AWS](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/throttling.html)
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/riseshia/sfn_job.

data/lib/active_job/queue_adapters/sfn_job_adapter.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  module QueueAdapters
+    class SfnJobAdapter
+      def enqueue(job)
+        SfnJob.enqueue(job)
+      end
+
+      def enqueue_at(job, timestamp)
+        raise NotImplementedError, "enqueue_at is not supported by SfnJob"
+      end
+
+      def enqueue_all(jobs)
+        raise NotImplementedError, "enqueue_all is not supported by SfnJob"
+      end
+    end
+  end
+end

data/lib/sfn_job/configuration.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module SfnJob
+  class Configuration
+    attr_accessor :region, :account_id, :stub_sfn_client
+
+    def initialize
+      @stub_sfn_client = false
+    end
+  end
+end

data/lib/sfn_job/railtie.rb

@@ -0,0 +1,11 @@
+ActiveSupport.on_load(:active_job) do
+  require 'active_job/queue_adapters/sfn_job_adapter'
+end
+
+module BarbequeClient
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load 'tasks/sfn_job.rake'
+    end
+  end
+end

data/lib/sfn_job/runner.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module SfnJob
+  class Runner
+    class << self
+      def run(serialized_job)
+        job_data = SfnJob.deserialize(serialized_job)
+        ActiveJob::Base.execute job_data
+      end
+    end
+  end
+end

data/lib/sfn_job/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module SfnJob
+  VERSION = "0.1.2"
+end

data/lib/sfn_job.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "json"
+require "aws-sdk-states"
+
+begin
+  require "rails"
+rescue LoadError
+else
+  require "sfn_job/railtie"
+end
+
+require_relative "sfn_job/version"
+require_relative "sfn_job/configuration"
+require_relative "sfn_job/runner"
+
+module SfnJob
+  MAX_SERIALIZED_JOB_SIZE = 60_000
+
+  class << self
+    def configure
+      yield config
+    end
+
+    def config
+      @config ||= Configuration.new
+    end
+
+    def enqueue(job)
+      sfn_client.start_execution({
+        state_machine_arn: build_arn(job.queue_name),
+        name: "#{job.class.name}-#{job.job_id}",
+        input: serialize(job),
+      })
+    end
+
+    def sfn_client
+      @sfn_client ||= Aws::States::Client.new(region: config.region, stub_responses: config.stub_sfn_client)
+    end
+
+    private def build_arn(sfn_name)
+      "arn:aws:states:#{config.region}:#{config.account_id}:stateMachine:#{sfn_name}"
+    end
+
+    private def serialize(job)
+      JSON.dump({ serialized_job: JSON.dump(job.serialize) }).tap do |serialized_job|
+        raise "Job serialization is too large" if serialized_job.bytesize > MAX_SERIALIZED_JOB_SIZE
+      end
+    end
+
+    def deserialize(serialized_job)
+      JSON.parse(serialized_job)
+    end
+  end
+end

data/lib/tasks/sfn_job.rake

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+namespace :sfn_job do
+  desc 'Start worker to execute jobs'
+  task execute: :environment do
+    require 'sfn_job'
+
+    $stdout.sync = true
+
+    if ENV['SERIALIZED_JOB'].nil?
+      raise 'SERIALIZED_JOB environment variable is required'
+    end
+
+    serialized_job = ENV.fetch('SERIALIZED_JOB')
+
+    SfnJob::Runner.run(serialized_job)
+  end
+end

metadata

@@ -0,0 +1,81 @@
+--- !ruby/object:Gem::Specification
+name: sfn_job
+version: !ruby/object:Gem::Version
+  version: 0.1.2
+platform: ruby
+authors:
+- Shia
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2024-11-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-states
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ActiveJob on State Functions
+email:
+- rise.shia@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/active_job/queue_adapters/sfn_job_adapter.rb
+- lib/sfn_job.rb
+- lib/sfn_job/configuration.rb
+- lib/sfn_job/railtie.rb
+- lib/sfn_job/runner.rb
+- lib/sfn_job/version.rb
+- lib/tasks/sfn_job.rake
+homepage: https://github.com/riseshia/sfn_job
+licenses: []
+metadata:
+  homepage_uri: https://github.com/riseshia/sfn_job
+  source_code_uri: https://github.com/riseshia/sfn_job
+  changelog_uri: https://github.com/riseshia/sfn_job
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.16
+signing_key:
+specification_version: 4
+summary: ActiveJob on State Functions
+test_files: []