med_pipe 0.1.0.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2d62e07ad3b1678e874749b5e34e930992476342d40ddad20be2dfd5b1da6593
+  data.tar.gz: 6d9c9dbffe01a6e7d4dd47c59a61f8233139d140600dc30d361cbe8d33c965ae
+SHA512:
+  metadata.gz: 198784f564acaa4e36cbfdfb8e8ded498645124ca6d6a74f07b32a6e0515f2fbb0ddb3a9df9c2c79e983f6cc3f3811c4aac5d03518237fdb4e1eaf9de6f36731
+  data.tar.gz: 51f1a3a9e7eaca4c62e304874022da628e4bf9c85de965c24328d12b5ea5c87abe0755f3b05e8f59a857d4d5615c869f058af9780e50f75af963118562ac67c5

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright mpg-taichi-sato
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,55 @@
+# MedPipe <sup>BETA</sup>
+100万 ~ 数10億程度のデータを処理するための仕組みを提供する Rails エンジンです。
+
+## Concept
+### MedPipe::Pipeline
+apply で後述する PipelineTask を登録し、run で順番に実行します。
+
+### MedPipe::PipelineTask
+Pipeline に登録する処理の単位です。  
+DB からの読み込みや、S3 へのアップロード等やることを分割してタスク化します。  
+大量データを扱う際には Enumerable::Lazy を使うことで分割して処理をすることができます。  
+call を実装する必要があります
+
+```.rb
+@param context [Hash] Stores data during pipeline execution
+@param prev_result [Object] The result of the previous task
+def call(context, prev_result)
+  yield 次のTaskに渡すデータ
+end
+```
+
+### MedPipe::PipelinePlan
+Pipeline の状態、オプション、結果を保存するためのモデルです。  
+Task で使うためのオプションを渡す方法は PipelinePlan から取得するか、contextで伝搬するかの二択です。
+
+### MedPipe::PipelineGroup
+一つのジョブで実行する Plan をまとめるためのモデルです。  
+実行中に parallel_limit を 0 にすることで中断することができます。
+
+## Usage
+
+1. Reader, Uploader 等の PipelineTask を作成 [Samples](https://github.com/medpeer-dev/med_pipe/tree/main/spec/dummy/app/models/pipeline_task)
+2. PipelineRunner を作成 [Sample](https://github.com/medpeer-dev/med_pipe/blob/main/spec/dummy/app/models/sample_pipeline_runner.rb)
+3. Pipeline を並列実行するためのジョブを作成 [Sample](https://github.com/medpeer-dev/med_pipe/blob/main/spec/dummy/app/jobs/sample_execute_pipeline_job.rb)
+4. PipelinePlan を登録するコードを記述
+5. 実行
+
+## Installation
+Add this line to your application's Gemfile:
+
+```ruby
+gem "med_pipe"
+```
+
+### migrationファイルの追加
+
+```shell
+$ rails med_pipe:install:migrations
+```
+
+## Contributing
+Bug reports and pull requests are welcome.
+
+## License
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require "bundler/setup"
+
+APP_RAKEFILE = File.expand_path("spec/dummy/Rakefile", __dir__)
+load "rails/tasks/engine.rake"
+
+load "rails/tasks/statistics.rake"
+
+require "bundler/gem_tasks"

data/app/models/med_pipe/application_record.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module MedPipe
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+
+    def self.table_name_prefix
+      "med_pipe_"
+    end
+  end
+end

data/app/models/med_pipe/pipeline_group.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class MedPipe::PipelineGroup < MedPipe::ApplicationRecord
+  has_many :pipeline_plans, class_name: "MedPipe::PipelinePlan", dependent: :destroy
+end

data/app/models/med_pipe/pipeline_plan.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class MedPipe::PipelinePlan < MedPipe::ApplicationRecord
+  belongs_to :pipeline_group, class_name: "MedPipe::PipelineGroup", optional: true
+
+  scope :active, -> { where(status: %i[enqueued running]) }
+
+  validates :name, presence: true
+  validates :output_unit, presence: true
+  validates :status, presence: true
+
+  # TODO: Rails6記法のため、Rails8に上げる際に定義の仕方を変える
+  # https://zenn.dev/kanazawa/articles/8bc1fcbba3ef1d#enum%E3%81%AE%E5%AE%9A%E7%BE%A9%E6%96%B9%E6%B3%95%E3%81%8C%E5%A4%89%E3%82%8F%E3%82%8B
+  enum status: {
+    waiting: "waiting",
+    enqueued: "enqueued",
+    running: "running",
+    finished: "finished",
+    failed: "failed"
+  }, _prefix: true
+
+  enum output_unit: {
+    daily: "daily",
+    all: "all"
+  }, _prefix: true
+end

data/config/routes.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+Rails.application.routes.draw do
+  # なし
+end

data/db/migrate/20241118063336_create_med_pipe_pipeline_plans.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class CreateMedPipePipelinePlans < ActiveRecord::Migration[7.2]
+  def change
+    create_table :med_pipe_pipeline_plans do |t|
+      t.string :name, null: false, comment: "パイプライン名"
+      t.integer :priority, null: false, default: 0, comment: "実行優先度"
+      t.string :status, null: false
+      t.string :output_unit, null: false, comment: "実行単位. 日ごと、全て等"
+      t.date :target_date, comment: "実行対象日. output_unit が daily の場合に指定"
+      t.bigint :data_count
+      t.string :file_name
+      t.bigint :file_size
+      t.string :upload_to
+      t.datetime :started_at
+      t.datetime :finished_at
+
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20241122022123_create_med_pipe_pipeline_groups.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class CreateMedPipePipelineGroups < ActiveRecord::Migration[7.2]
+  def change
+    create_table :med_pipe_pipeline_groups do |t|
+      t.integer :parallel_limit, null: false, default: 1, comment: "並列実行数"
+      t.timestamps
+    end
+
+    add_reference :med_pipe_pipeline_plans, :pipeline_group, null: false
+  end
+end

data/lib/med_pipe/batch_id_fetcher.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+# idを最大max_load_size件ずつ分割取得するためのクラス
+# 使い時:
+# - 10万件以上のidを取得したい場合
+# - 速度を改善するために in_batches を使いたくない場合
+class MedPipe::BatchIdFetcher
+  def initialize(relation, batch_size: 1_000, max_load_size: 100_000)
+    @relation = relation
+    @batch_size = batch_size
+    @max_load_size = max_load_size
+    validate_parameters
+  end
+
+  def each
+    return enum_for(:each) unless block_given?
+
+    last_id = 0
+    cached_ids = []
+
+    loop do
+      loaded_ids = load_ids(last_id)
+      break if loaded_ids.blank?
+
+      last_id = loaded_ids.last
+      cached_ids.concat(loaded_ids)
+      yield(cached_ids.shift(@batch_size)) while cached_ids.size >= @batch_size
+
+      if loaded_ids.size < @max_load_size
+        yield(cached_ids) if cached_ids.present?
+        break
+      end
+    end
+  end
+
+  private
+
+  def validate_parameters
+    raise ArgumentError, "batch_size must be greater than 0" if @batch_size <= 0
+  end
+
+  def load_ids(last_id)
+    if last_id.zero?
+      @relation.limit(@max_load_size).order(:id).ids
+    else
+      @relation.where("id > ?", last_id).order(:id).limit(@max_load_size).ids
+    end
+  end
+end

data/lib/med_pipe/batch_reader.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+# 大量データを分割取得するためのクラス
+# in_batches では scope が全クエリに含まれるが、本クラスではidの取得でのみ scope を使用する
+class MedPipe::BatchReader
+  def initialize(model_class, scope: nil, pluck_columns: [:id], batch_size: 1_000,
+                 max_id_load_size: 100_000)
+    @model_class = model_class
+    @scope = scope || model_class.all
+    @pluck_columns = pluck_columns
+    @batch_size = batch_size
+    @max_id_load_size = max_id_load_size
+    @around_load_callback = nil
+    validate_parameters
+  end
+
+  # EXAMPLE:
+  #   MedPipe::BatchReader.new(User)
+  #     .around_load { |&block| ApplicationRecord.connected_to(role: :reading, &block) }
+  def around_load(&block)
+    @around_load_callback = block
+    self
+  end
+
+  # @yieldparam [Array] pluck結果を1件ずつ渡す
+  def each(&block)
+    return enum_for(:each) unless block
+
+    each_ids = MedPipe::BatchIdFetcher.new(@scope, batch_size: @batch_size, max_load_size: @max_id_load_size).each
+    loop do
+      records = @around_load_callback&.call { batch_load(each_ids) } || batch_load(each_ids)
+      records.each(&block)
+    rescue StopIteration
+      break
+    end
+  end
+
+  private
+
+  def validate_parameters
+    raise ArgumentError, "model_class must be a subclass of ApplicationRecord" unless @model_class < ApplicationRecord
+  end
+
+  def batch_load(each_ids)
+    # in_batches ではクエリキャッシュが無効になっているため、それに倣う
+    @model_class.uncached do
+      @model_class.where(id: each_ids.next).pluck(*@pluck_columns)
+    end
+  end
+end

data/lib/med_pipe/engine.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class MedPipe::Engine < Rails::Engine
+  # migrationファイルの生成コマンドを med_pipe_engine:install:migrations から med_pipe:install:migrations に変更
+  # https://edgeapi.rubyonrails.org/classes/Rails/Engine.html#class-Rails::Engine-label-Engine+name
+  engine_name "med_pipe"
+end

data/lib/med_pipe/pipeline.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# 直列に繋いだtaskを順番に実行するクラス
+class MedPipe::Pipeline
+  def initialize
+    @tasks = []
+  end
+
+  # @param task [Object] def call(context, prev_result, &block) を実装したクラス
+  def apply(task)
+    @tasks << task
+    self
+  end
+
+  # @param context [Hash] Stores data during pipeline execution
+  def run(context = {}) = run_task_recursive(context)
+  # 展開すると以下のようになる
+  # @tasks[0].call(context, nil) do |prev_result|
+  #   @tasks[1].call(context, prev_result) do |prev_result|
+  #     @tasks[2].call(context, prev_result) do |prev_result|
+  #       nil
+  #     end
+  #   end
+  # end
+
+  private
+
+  def run_task_recursive(context, prev_result = nil, task_index = 0)
+    return prev_result if task_index >= @tasks.size
+
+    @tasks[task_index]&.call(context, prev_result) do |result|
+      run_task_recursive(context, result, task_index + 1)
+    end
+  end
+end

data/lib/med_pipe/pipeline_plan_consumer.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+# enqueued な pipeline plan を1つ取得 & 実行
+class MedPipe::PipelinePlanConsumer
+  # @param [Proc] pipeline_runner pipeline plan から pipeline を作成し実行する
+  def initialize(pipeline_group:, pipeline_runner:)
+    @pipeline_group = pipeline_group
+    @pipeline_runner = pipeline_runner
+  end
+
+  # @return [PipelinePlan] 実行した pipeline plan。なければ nil
+  def run
+    pipeline_plan = fetch_and_run_pipeline_plan
+    return nil if pipeline_plan.nil?
+
+    @pipeline_runner.call(pipeline_plan)
+    complete_pipeline_plan(pipeline_plan)
+    pipeline_plan
+  rescue StandardError => e
+    error_pipeline_plan(pipeline_plan)
+    raise e
+  end
+
+  private
+
+  def fetch_and_run_pipeline_plan
+    ApplicationRecord.transaction do
+      target_pipeline_plan = @pipeline_group.pipeline_plans.lock.status_enqueued.order(priority: :desc).first
+      return if target_pipeline_plan.nil?
+
+      target_pipeline_plan.update!(status: :running)
+      target_pipeline_plan
+    end
+  end
+
+  def complete_pipeline_plan(pipeline_plan)
+    pipeline_plan.update!(status: :finished, finished_at: Time.current)
+  end
+
+  def error_pipeline_plan(pipeline_plan)
+    pipeline_plan.update!(status: :failed)
+  end
+end

data/lib/med_pipe/pipeline_plan_producer.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+# prioryty が高いものから順に、並列数を踏まえて複数のPipelinePlanの状態をenqueuedに変更する
+class MedPipe::PipelinePlanProducer
+  # @param pipeline_group [MedPipe::PipelineGroup]
+  def initialize(pipeline_group)
+    @pipeline_group = pipeline_group
+  end
+
+  # @return [Array<MedPipe::PipelinePlan>] Enqueued pipeline plans. 未実行ならnilを返す
+  def run
+    return if @pipeline_group.parallel_limit <= 0
+
+    @pipeline_group.with_lock do
+      enqueue_count = @pipeline_group.parallel_limit - @pipeline_group.pipeline_plans.active.count
+      enqueue(enqueue_count) if enqueue_count.positive?
+    end
+  end
+
+  private
+
+  def enqueue(size)
+    target_pipeline_plans = fetch_target_pipeline_plans(size: size)
+    return if target_pipeline_plans.empty?
+
+    target_pipeline_plans.each do |pipline_plan|
+      pipline_plan.update!(status: :enqueued)
+    end
+  end
+
+  def fetch_target_pipeline_plans(size:)
+    @pipeline_group.pipeline_plans.status_waiting.order(priority: :desc).limit(size)
+  end
+end

data/lib/med_pipe/pipeline_runner_base.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+# PipelinePlanConsumerに渡すPipelineRunnerの作成を補助するクラス
+# call(pipeline_plan)さえ実装していれば良いため、必ずしも本クラスを使う必要はありません。
+class MedPipe::PipelineRunnerBase
+  # PipelinePlanConsumerから呼び出されるメソッド
+  def call(pipeline_plan)
+    pipeline = build_pipeline(pipeline_plan)
+    context = { plan: pipeline_plan }
+    pipeline.run(context)
+  end
+
+  def build_pipeline(pipeline_plan)
+    raise NotImplementedError("#{pipeline_plan.name}に対応するPipelineを作成する処理をサブクラスで実装してください")
+  end
+end

data/lib/med_pipe/pipeline_task/counter.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class MedPipe::PipelineTask::Counter
+  def initialize
+    @count = 0
+  end
+
+  # @param context [Hash]
+  # @param input [Enumerable<Array<Object>>]
+  # @yieldparam [Enumerable<Array<Object>>] inputをそのまま流す
+  def call(context, input)
+    yield input.map { |x| increment(context); x } # rubocop:disable Style/Semicolon
+  end
+
+  def increment(context)
+    @count += 1
+    context[:data_count] = @count
+  end
+end

data/lib/med_pipe/pipeline_task/plan_updater.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+class MedPipe::PipelineTask::PlanUpdater
+  # @param save [Boolean] trueの場合、Planを保存する。finishにするために更新が走るためここで保存しないことをdefaultにしている
+  def initialize(save: false)
+    @save = save
+  end
+
+  # @param context [Hash]
+  # @param input [Enumerable<Array<Object>>]
+  # @yieldparam [Enumerable<Array<Object>>] inputをそのまま流す
+  def call(context, input)
+    update_plan(context)
+    block_given? ? yield(input) : input
+  end
+
+  private
+
+  def update_plan(context)
+    return unless context[:plan]
+
+    plan = context[:plan]
+    plan.data_count = context[:data_count] if context[:data_count]
+    plan.file_name = context[:file_name] if context[:file_name]
+    plan.file_size = context[:file_size] if context[:file_size]
+    plan.upload_to = context[:upload_to] if context[:upload_to]
+    plan.save if @save
+  end
+end

data/lib/med_pipe/pipeline_task/tsv_generator.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "csv"
+
+class MedPipe::PipelineTask::TsvGenerater
+  TSV_OPTION = { col_sep: "\t" }.freeze
+
+  # @param lines [Enumerable<Array<Object>>] to_s可能なオブジェクトの配列のEnumerable
+  # @yieldparam [File] 生成したtsvファイル
+  def call(_context, lines)
+    Tempfile.create do |file|
+      lines.each do |line|
+        # nil に置き換えることで""という文字列が出力されてしまうのを回避
+        normalized_line = line.map { |v| v == "" ? nil : v }
+        tsv_line = CSV.generate_line(normalized_line, **TSV_OPTION)
+        file.puts(tsv_line)
+      end
+      file.rewind
+
+      yield(file)
+    end
+  end
+end

data/lib/med_pipe/pipeline_task.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+module MedPipe::PipelineTask
+end

data/lib/med_pipe/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module MedPipe
+  VERSION = "0.1.0.4"
+end

data/lib/med_pipe.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "med_pipe/version"
+require "med_pipe/engine"
+
+require "med_pipe/batch_id_fetcher"
+require "med_pipe/batch_reader"
+require "med_pipe/pipeline_plan_consumer"
+require "med_pipe/pipeline_plan_producer"
+require "med_pipe/pipeline_runner_base"
+require "med_pipe/pipeline"
+
+require "med_pipe/pipeline_task"
+require "med_pipe/pipeline_task/counter"
+require "med_pipe/pipeline_task/tsv_generator"
+require "med_pipe/pipeline_task/plan_updater"
+
+module MedPipe
+  # Your code goes here...
+end

data/lib/tasks/med_pipe_tasks.rake

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+# desc "Explaining what the task does"
+# task :med_pipe do
+#   # Task goes here
+# end

data/spec/factories/med_pipe_pipeline_groups.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :med_pipe_pipeline_group, class: "MedPipe::PipelineGroup" do
+    parallel_limit { 1 }
+  end
+end

data/spec/factories/med_pipe_pipeline_plans.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :med_pipe_pipeline_plan, class: "MedPipe::PipelinePlan" do
+    name { "dummy" }
+    output_unit { :all }
+    status { :waiting }
+    association :pipeline_group, factory: :med_pipe_pipeline_group
+  end
+end

metadata

@@ -0,0 +1,107 @@
+--- !ruby/object:Gem::Specification
+name: med_pipe
+version: !ruby/object:Gem::Version
+  version: 0.1.0.4
+platform: ruby
+authors:
+- mpg-taichi-sato
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2024-11-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.1.7
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '8.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.1.7
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '8.0'
+description: "# MedPipe <sup>BETA</sup>\n100万 ~ 数10億程度のデータを処理するための仕組みを提供する Rails エンジンです。\n\n##
+  Concept\n### MedPipe::Pipeline\napply で後述する PipelineTask を登録し、run で順番に実行します。\n\n###
+  MedPipe::PipelineTask\nPipeline に登録する処理の単位です。  \nDB からの読み込みや、S3 へのアップロード等やることを分割してタスク化します。
+  \ \n大量データを扱う際には Enumerable::Lazy を使うことで分割して処理をすることができます。  \ncall を実装する必要があります\n\n```.rb\n@param
+  context [Hash] Stores data during pipeline execution\n@param prev_result [Object]
+  The result of the previous task\ndef call(context, prev_result)\n  yield 次のTaskに渡すデータ\nend\n```\n\n###
+  MedPipe::PipelinePlan\nPipeline の状態、オプション、結果を保存するためのモデルです。  \nTask で使うためのオプションを渡す方法は
+  PipelinePlan から取得するか、contextで伝搬するかの二択です。\n\n### MedPipe::PipelineGroup\n一つのジョブで実行する
+  Plan をまとめるためのモデルです。  \n実行中に parallel_limit を 0 にすることで中断することができます。\n\n## Usage\n\n1.
+  Reader, Uploader 等の PipelineTask を作成 [Samples](https://github.com/medpeer-dev/med_pipe/tree/main/spec/dummy/app/models/pipeline_task)\n2.
+  PipelineRunner を作成 [Sample](https://github.com/medpeer-dev/med_pipe/blob/main/spec/dummy/app/models/sample_pipeline_runner.rb)\n3.
+  Pipeline を並列実行するためのジョブを作成 [Sample](https://github.com/medpeer-dev/med_pipe/blob/main/spec/dummy/app/jobs/sample_execute_pipeline_job.rb)\n4.
+  PipelinePlan を登録するコードを記述\n5. 実行\n\n## Installation\nAdd this line to your application's
+  Gemfile:\n\n```ruby\ngem \"med_pipe\"\n```\n\n### migrationファイルの追加\n\n```shell\n$
+  rails med_pipe:install:migrations\n```\n\n## Contributing\nBug reports and pull
+  requests are welcome.\n\n## License\nThe gem is available as open source under the
+  terms of the [MIT License](https://opensource.org/licenses/MIT).\n"
+email: 
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- README.md
+- Rakefile
+- app/models/med_pipe/application_record.rb
+- app/models/med_pipe/pipeline_group.rb
+- app/models/med_pipe/pipeline_plan.rb
+- config/routes.rb
+- db/migrate/20241118063336_create_med_pipe_pipeline_plans.rb
+- db/migrate/20241122022123_create_med_pipe_pipeline_groups.rb
+- lib/med_pipe.rb
+- lib/med_pipe/batch_id_fetcher.rb
+- lib/med_pipe/batch_reader.rb
+- lib/med_pipe/engine.rb
+- lib/med_pipe/pipeline.rb
+- lib/med_pipe/pipeline_plan_consumer.rb
+- lib/med_pipe/pipeline_plan_producer.rb
+- lib/med_pipe/pipeline_runner_base.rb
+- lib/med_pipe/pipeline_task.rb
+- lib/med_pipe/pipeline_task/counter.rb
+- lib/med_pipe/pipeline_task/plan_updater.rb
+- lib/med_pipe/pipeline_task/tsv_generator.rb
+- lib/med_pipe/version.rb
+- lib/tasks/med_pipe_tasks.rake
+- spec/factories/med_pipe_pipeline_groups.rb
+- spec/factories/med_pipe_pipeline_plans.rb
+homepage: https://github.com/medpeer-dev/med_pipe
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/medpeer-dev/med_pipe
+  source_code_uri: https://github.com/medpeer-dev/med_pipe
+  rubygems_mfa_required: 'true'
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.22
+signing_key: 
+specification_version: 4
+summary: Provides a system for processing data ranging from 1 million to several billion
+  records
+test_files: []