dynflow 1.6.11 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: db2725603e440298025d434778ebf2e218e6c3e30fb7db07aaddea80648cbb2d
-  data.tar.gz: 4d8003de94bec301bd411c1c3ee92df9e1d10cc45bc36d4d843a75c2eb664171
+  metadata.gz: b1fd4cb9db86befb0bfc96cd5c98f2ca9aa0f6f40249712cbbb8b250b9b9a142
+  data.tar.gz: 218d8a4ce5ec717f24ca4d5d730a5c0620f65ab00e6a35e0595a18876f4b9a31
 SHA512:
-  metadata.gz: ab0fa706aa8b41c1197aef44bf57a87f4e5c37d7d343a9ca7e83e20af8de89b4e0fb2e72e7a1b93fffa05ac3265c78a48d598d7b8e0292785cd2276057edd0f0
-  data.tar.gz: 07e82b78806a61511b0ddff0240dd596172dfe79735bc8b931d7972b4d322d29bca1686be8be0087a105171e7348c0ee24cfdb2b01cc1a8dcf8d97ad8b0134c8
+  metadata.gz: 80b8644418108a107c2a73481fb793647ac6ce40f9789b4efa064970bd6cccd54579ce95ff0b5a4710f033e4defaa7d0fd50ec96d9c51e91666adcd875cc9cc4
+  data.tar.gz: 6a28661cdfeff6555c970df957f10f28d4c23baf977b1212cf3253bac973cef967b1c40be7b77dbaa3fe6811e484be319a04417a0895dd81a32edd534ce58975

data/.github/workflows/ruby.yml

@@ -35,10 +35,9 @@ jobs:
       fail-fast: false
       matrix:
         ruby_version:
-          - 2.5.0
-          - 2.6.0
           - 2.7.0
           - 3.0.0
+          - 3.2.0
         concurrent_ruby_ext:
           - 'true'
           - 'false'
@@ -54,30 +53,23 @@ jobs:
           - db: sqlite3
             conn_string: sqlite:/
         exclude:
-          - db: mysql
-            ruby_version: 2.5.0
-          - db: mysql
-            ruby_version: 2.6.0
           - db: mysql
             ruby_version: 3.0.0
+          - db: mysql
+            ruby_version: 3.2.0
           - db: mysql
             concurrent_ruby_ext: 'true'
-          - db: sqlite3
-            ruby_version: 2.5.0
-          - db: sqlite3
-            ruby_version: 2.6.0
           - db: sqlite3
             ruby_version: 3.0.0
+          - db: sqlite3
+            ruby_version: 3.2.0
           - db: sqlite3
             concurrent_ruby_ext: 'true'
           - db: postgresql
-            ruby_version: 2.5.0
-            concurrent_ruby_ext: 'true'
-          - db: postgresql
-            ruby_version: 2.6.0
+            ruby_version: 3.0.0
             concurrent_ruby_ext: 'true'
           - db: postgresql
-            ruby_version: 3.0.0
+            ruby_version: 3.2.0
             concurrent_ruby_ext: 'true'
 
     services:

data/lib/dynflow/action.rb

@@ -179,6 +179,12 @@ module Dynflow
       @output_chunks ||= world.persistence.load_output_chunks(@execution_plan_id, @id)
     end
 
+    def drop_output_chunks!
+      @pending_output_chunks = []
+      @output_chunks = []
+      world.persistence.delete_output_chunks(@execution_plan_id, @id)
+    end
+
     def caller_action
       phase! Present
       return nil if @caller_action_id

data/lib/dynflow/persistence.rb

@@ -56,12 +56,20 @@ module Dynflow
       adapter.load_output_chunks(execution_plan_id, action_id)
     end
 
+    def delete_output_chunks(execution_plan_id, action_id)
+      adapter.delete_output_chunks(execution_plan_id, action_id)
+    end
+
     def find_execution_plans(options)
       adapter.find_execution_plans(options).map do |execution_plan_hash|
         ExecutionPlan.new_from_hash(execution_plan_hash, @world)
       end
     end
 
+    def find_execution_plan_statuses(options)
+      adapter.find_execution_plan_statuses(options)
+    end
+
     def find_execution_plan_counts(options)
       adapter.find_execution_plan_counts(options)
     end

data/lib/dynflow/persistence_adapters/abstract.rb

@@ -46,6 +46,10 @@ module Dynflow
         filter(:execution_plan, options[:filters]).count
       end
 
+      def find_execution_plan_statuses(options)
+        raise NotImplementedError
+      end
+
       # @param filters [Hash{ String => Object }] filters to determine
       #   what to delete
       # @param batch_size the size of the chunks to iterate over when
@@ -104,6 +108,18 @@ module Dynflow
         raise NotImplementedError
       end
 
+      def save_output_chunks(execution_plan_id, action_id, chunks)
+        raise NotImplementedError
+      end
+
+      def load_output_chunks(execution_plan_id, action_id)
+        raise NotImplementedError
+      end
+
+      def delete_output_chunks(execution_plan_id, action_id)
+        raise NotImplementedError
+      end
+
       # for debug purposes
       def to_hash
         raise NotImplementedError

data/lib/dynflow/persistence_adapters/sequel.rb

@@ -78,6 +78,16 @@ module Dynflow
         filter(:execution_plan, table(:execution_plan), options[:filters]).count
       end
 
+      def find_execution_plan_statuses(options)
+        plans = filter(:execution_plan, table(:execution_plan), options[:filters])
+                  .select(:uuid, :state, :result)
+
+        plans.each_with_object({}) do |current, acc|
+          uuid = current.delete(:uuid)
+          acc[uuid] = current
+        end
+      end
+
       def delete_execution_plans(filters, batch_size = 1000, backup_dir = nil)
         count = 0
         filter(:execution_plan, table(:execution_plan), filters).each_slice(batch_size) do |plans|
@@ -190,6 +200,10 @@ module Dynflow
         load_records :output_chunk, { execution_plan_uuid: execution_plan_id, action_id: action_id }, [:timestamp, :kind, :chunk]
       end
 
+      def delete_output_chunks(execution_plan_id, action_id)
+        filter(:output_chunk, table(:output_chunk), { execution_plan_uuid: execution_plan_id, action_id: action_id }).delete
+      end
+
       def connector_feature!
         unless @additional_responsibilities[:connector]
           raise "The sequel persistence adapter connector feature used but not enabled in additional_features"

data/lib/dynflow/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Dynflow
-  VERSION = '1.6.11'
+  VERSION = '1.7.0'
 end

data/test/action_test.rb

@@ -889,5 +889,48 @@ module Dynflow
         end
       end
     end
+
+    describe 'output chunks' do
+      include ::Dynflow::Testing::Factories
+
+      class OutputChunkAction < ::Dynflow::Action
+        def run(event = nil)
+          output[:counter] ||= 0
+          case event
+          when nil
+            output_chunk("Chunk #{output[:counter]}")
+            output[:counter] += 1
+            suspend
+          when :exit
+            return
+          end
+        end
+
+        def finalize
+          drop_output_chunks!
+        end
+      end
+
+      it 'collects and drops output chunks' do
+        action = create_and_plan_action(OutputChunkAction)
+        _(action.pending_output_chunks).must_equal nil
+
+        action = run_action(action)
+        _(action.pending_output_chunks.count).must_equal 1
+
+        action = run_action(action)
+        _(action.pending_output_chunks.count).must_equal 2
+
+        action = run_action(action, :exit)
+        _(action.pending_output_chunks.count).must_equal 2
+
+        persistence = mock()
+        persistence.expects(:delete_output_chunks).with(action.execution_plan_id, action.id)
+        action.world.stubs(:persistence).returns(persistence)
+
+        action = finalize_action(action)
+        _(action.pending_output_chunks.count).must_equal 0
+      end
+    end
   end
 end

data/test/persistence_test.rb

@@ -159,6 +159,46 @@ module Dynflow
           end
         end
 
+        describe '#def find_execution_plan_statuses' do
+          before do
+            # the tests expect clean field
+            adapter.delete_execution_plans({})
+          end
+
+          it 'supports filtering' do
+            prepare_and_save_plans
+            if adapter.ordering_by.include?('state')
+              loaded_plans = adapter.find_execution_plan_statuses(filters: { label: ['test1'] })
+              _(loaded_plans).must_equal({ 'plan1' => { state: 'paused', result: nil} })
+
+              loaded_plans = adapter.find_execution_plan_statuses(filters: { state: ['paused'] })
+              _(loaded_plans).must_equal({"plan1"=>{:state=>"paused", :result=>nil},
+                                          "plan3"=>{:state=>"paused", :result=>nil},
+                                          "plan4"=>{:state=>"paused", :result=>nil}})
+
+              loaded_plans = adapter.find_execution_plan_statuses(filters: { state: ['stopped'] })
+              _(loaded_plans).must_equal({"plan2"=>{:state=>"stopped", :result=>nil}})
+
+              loaded_plans = adapter.find_execution_plan_statuses(filters: { state: [] })
+              _(loaded_plans).must_equal({})
+
+              loaded_plans = adapter.find_execution_plan_statuses(filters: { state: ['stopped', 'paused'] })
+              _(loaded_plans).must_equal({"plan1"=>{:state=>"paused", :result=>nil},
+                                          "plan2"=>{:state=>"stopped", :result=>nil},
+                                          "plan3"=>{:state=>"paused", :result=>nil}, "plan4"=>{:state=>"paused", :result=>nil}})
+
+              loaded_plans = adapter.find_execution_plan_statuses(filters: { 'state' => ['stopped', 'paused'] })
+              _(loaded_plans).must_equal({"plan1"=>{:state=>"paused", :result=>nil},
+                                          "plan2"=>{:state=>"stopped", :result=>nil},
+                                          "plan3"=>{:state=>"paused", :result=>nil},
+                                          "plan4"=>{:state=>"paused", :result=>nil}})
+
+              loaded_plans = adapter.find_execution_plan_statuses(filters: { label: ['test1'], :delayed => true })
+              _(loaded_plans).must_equal({})
+            end
+          end
+        end
+
         describe '#def find_execution_plan_counts' do
           before do
             # the tests expect clean field
@@ -331,6 +371,20 @@ module Dynflow
             _(plans.first[:execution_plan_uuid]).must_equal 'plan1'
           end
         end
+
+        describe '#delete_output_chunks' do
+          it 'deletes output chunks' do
+            prepare_plans_with_actions
+
+            adapter.save_output_chunks('plan1', 1, [{chunk: "Hello", timestamp: Time.now}, {chunk: "Bye", timestamp: Time.now}])
+            chunks = adapter.load_output_chunks('plan1', 1)
+            _(chunks.length).must_equal 2
+            deleted = adapter.delete_output_chunks('plan1', 1)
+            _(deleted).must_equal 2
+            chunks = adapter.load_output_chunks('plan1', 1)
+            _(chunks.length).must_equal 0
+          end
+        end
       end
 
       describe Dynflow::PersistenceAdapters::Sequel do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dynflow
 version: !ruby/object:Gem::Version
-  version: 1.6.11
+  version: 1.7.0
 platform: ruby
 authors:
 - Ivan Necas
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-05-09 00:00:00.000000000 Z
+date: 2023-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json