dynflow 1.6.10 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 79387ebdc9ad0bbae88897885da885483c4d57e08af9be69f6f2369be67bbe5b
-  data.tar.gz: aa5413ff3913decab84dc70e86f94f4293acfcc16340e7ad6216abf9943335b1
+  metadata.gz: b1fd4cb9db86befb0bfc96cd5c98f2ca9aa0f6f40249712cbbb8b250b9b9a142
+  data.tar.gz: 218d8a4ce5ec717f24ca4d5d730a5c0620f65ab00e6a35e0595a18876f4b9a31
 SHA512:
-  metadata.gz: 99132ff502c108b47595d211caba78c3034adb05390937269814d09a8da67c546b9bc2e7a2c776bcba0596d76ee50b7229c7497aad77e2d2483c43351d4a65a5
-  data.tar.gz: 52e2bb51d1fdc668c8837646635c37c58c65761d5ef3812294838dfacc54dfc48ae26550bd132a077dfc8579ee4fa86787fe17d27f45389a6a536328276de897
+  metadata.gz: 80b8644418108a107c2a73481fb793647ac6ce40f9789b4efa064970bd6cccd54579ce95ff0b5a4710f033e4defaa7d0fd50ec96d9c51e91666adcd875cc9cc4
+  data.tar.gz: 6a28661cdfeff6555c970df957f10f28d4c23baf977b1212cf3253bac973cef967b1c40be7b77dbaa3fe6811e484be319a04417a0895dd81a32edd534ce58975

data/.github/workflows/ruby.yml

@@ -35,10 +35,9 @@ jobs:
       fail-fast: false
       matrix:
         ruby_version:
-          - 2.5.0
-          - 2.6.0
           - 2.7.0
           - 3.0.0
+          - 3.2.0
         concurrent_ruby_ext:
           - 'true'
           - 'false'
@@ -54,30 +53,23 @@ jobs:
           - db: sqlite3
             conn_string: sqlite:/
         exclude:
-          - db: mysql
-            ruby_version: 2.5.0
-          - db: mysql
-            ruby_version: 2.6.0
           - db: mysql
             ruby_version: 3.0.0
+          - db: mysql
+            ruby_version: 3.2.0
           - db: mysql
             concurrent_ruby_ext: 'true'
-          - db: sqlite3
-            ruby_version: 2.5.0
-          - db: sqlite3
-            ruby_version: 2.6.0
           - db: sqlite3
             ruby_version: 3.0.0
+          - db: sqlite3
+            ruby_version: 3.2.0
           - db: sqlite3
             concurrent_ruby_ext: 'true'
           - db: postgresql
-            ruby_version: 2.5.0
-            concurrent_ruby_ext: 'true'
-          - db: postgresql
-            ruby_version: 2.6.0
+            ruby_version: 3.0.0
             concurrent_ruby_ext: 'true'
           - db: postgresql
-            ruby_version: 3.0.0
+            ruby_version: 3.2.0
             concurrent_ruby_ext: 'true'
 
     services:

data/lib/dynflow/action.rb

@@ -179,6 +179,12 @@ module Dynflow
       @output_chunks ||= world.persistence.load_output_chunks(@execution_plan_id, @id)
     end
 
+    def drop_output_chunks!
+      @pending_output_chunks = []
+      @output_chunks = []
+      world.persistence.delete_output_chunks(@execution_plan_id, @id)
+    end
+
     def caller_action
       phase! Present
       return nil if @caller_action_id

data/lib/dynflow/persistence.rb

@@ -56,12 +56,20 @@ module Dynflow
       adapter.load_output_chunks(execution_plan_id, action_id)
     end
 
+    def delete_output_chunks(execution_plan_id, action_id)
+      adapter.delete_output_chunks(execution_plan_id, action_id)
+    end
+
     def find_execution_plans(options)
       adapter.find_execution_plans(options).map do |execution_plan_hash|
         ExecutionPlan.new_from_hash(execution_plan_hash, @world)
       end
     end
 
+    def find_execution_plan_statuses(options)
+      adapter.find_execution_plan_statuses(options)
+    end
+
     def find_execution_plan_counts(options)
       adapter.find_execution_plan_counts(options)
     end

data/lib/dynflow/persistence_adapters/abstract.rb

@@ -46,6 +46,10 @@ module Dynflow
         filter(:execution_plan, options[:filters]).count
       end
 
+      def find_execution_plan_statuses(options)
+        raise NotImplementedError
+      end
+
       # @param filters [Hash{ String => Object }] filters to determine
       #   what to delete
       # @param batch_size the size of the chunks to iterate over when
@@ -104,6 +108,18 @@ module Dynflow
         raise NotImplementedError
       end
 
+      def save_output_chunks(execution_plan_id, action_id, chunks)
+        raise NotImplementedError
+      end
+
+      def load_output_chunks(execution_plan_id, action_id)
+        raise NotImplementedError
+      end
+
+      def delete_output_chunks(execution_plan_id, action_id)
+        raise NotImplementedError
+      end
+
       # for debug purposes
       def to_hash
         raise NotImplementedError

data/lib/dynflow/persistence_adapters/sequel.rb

@@ -78,6 +78,16 @@ module Dynflow
         filter(:execution_plan, table(:execution_plan), options[:filters]).count
       end
 
+      def find_execution_plan_statuses(options)
+        plans = filter(:execution_plan, table(:execution_plan), options[:filters])
+                  .select(:uuid, :state, :result)
+
+        plans.each_with_object({}) do |current, acc|
+          uuid = current.delete(:uuid)
+          acc[uuid] = current
+        end
+      end
+
       def delete_execution_plans(filters, batch_size = 1000, backup_dir = nil)
         count = 0
         filter(:execution_plan, table(:execution_plan), filters).each_slice(batch_size) do |plans|
@@ -190,6 +200,10 @@ module Dynflow
         load_records :output_chunk, { execution_plan_uuid: execution_plan_id, action_id: action_id }, [:timestamp, :kind, :chunk]
       end
 
+      def delete_output_chunks(execution_plan_id, action_id)
+        filter(:output_chunk, table(:output_chunk), { execution_plan_uuid: execution_plan_id, action_id: action_id }).delete
+      end
+
       def connector_feature!
         unless @additional_responsibilities[:connector]
           raise "The sequel persistence adapter connector feature used but not enabled in additional_features"
@@ -393,11 +407,11 @@ module Dynflow
         hash = if record[:data].nil?
                  SERIALIZABLE_COLUMNS.fetch(what, []).each do |key|
                    key = key.to_sym
-                   record[key] = MessagePack.unpack((record[key])) unless record[key].nil?
+                   record[key] = MessagePack.unpack(record[key].to_s) unless record[key].nil?
                  end
                  record
                else
-                 MessagePack.unpack(record[:data])
+                 MessagePack.unpack(record[:data].to_s)
                end
         Utils.indifferent_hash(hash)
       end

data/lib/dynflow/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Dynflow
-  VERSION = '1.6.10'
+  VERSION = '1.7.0'
 end

data/test/action_test.rb

@@ -889,5 +889,48 @@ module Dynflow
         end
       end
     end
+
+    describe 'output chunks' do
+      include ::Dynflow::Testing::Factories
+
+      class OutputChunkAction < ::Dynflow::Action
+        def run(event = nil)
+          output[:counter] ||= 0
+          case event
+          when nil
+            output_chunk("Chunk #{output[:counter]}")
+            output[:counter] += 1
+            suspend
+          when :exit
+            return
+          end
+        end
+
+        def finalize
+          drop_output_chunks!
+        end
+      end
+
+      it 'collects and drops output chunks' do
+        action = create_and_plan_action(OutputChunkAction)
+        _(action.pending_output_chunks).must_equal nil
+
+        action = run_action(action)
+        _(action.pending_output_chunks.count).must_equal 1
+
+        action = run_action(action)
+        _(action.pending_output_chunks.count).must_equal 2
+
+        action = run_action(action, :exit)
+        _(action.pending_output_chunks.count).must_equal 2
+
+        persistence = mock()
+        persistence.expects(:delete_output_chunks).with(action.execution_plan_id, action.id)
+        action.world.stubs(:persistence).returns(persistence)
+
+        action = finalize_action(action)
+        _(action.pending_output_chunks.count).must_equal 0
+      end
+    end
   end
 end

data/test/persistence_test.rb

@@ -159,6 +159,46 @@ module Dynflow
           end
         end
 
+        describe '#def find_execution_plan_statuses' do
+          before do
+            # the tests expect clean field
+            adapter.delete_execution_plans({})
+          end
+
+          it 'supports filtering' do
+            prepare_and_save_plans
+            if adapter.ordering_by.include?('state')
+              loaded_plans = adapter.find_execution_plan_statuses(filters: { label: ['test1'] })
+              _(loaded_plans).must_equal({ 'plan1' => { state: 'paused', result: nil} })
+
+              loaded_plans = adapter.find_execution_plan_statuses(filters: { state: ['paused'] })
+              _(loaded_plans).must_equal({"plan1"=>{:state=>"paused", :result=>nil},
+                                          "plan3"=>{:state=>"paused", :result=>nil},
+                                          "plan4"=>{:state=>"paused", :result=>nil}})
+
+              loaded_plans = adapter.find_execution_plan_statuses(filters: { state: ['stopped'] })
+              _(loaded_plans).must_equal({"plan2"=>{:state=>"stopped", :result=>nil}})
+
+              loaded_plans = adapter.find_execution_plan_statuses(filters: { state: [] })
+              _(loaded_plans).must_equal({})
+
+              loaded_plans = adapter.find_execution_plan_statuses(filters: { state: ['stopped', 'paused'] })
+              _(loaded_plans).must_equal({"plan1"=>{:state=>"paused", :result=>nil},
+                                          "plan2"=>{:state=>"stopped", :result=>nil},
+                                          "plan3"=>{:state=>"paused", :result=>nil}, "plan4"=>{:state=>"paused", :result=>nil}})
+
+              loaded_plans = adapter.find_execution_plan_statuses(filters: { 'state' => ['stopped', 'paused'] })
+              _(loaded_plans).must_equal({"plan1"=>{:state=>"paused", :result=>nil},
+                                          "plan2"=>{:state=>"stopped", :result=>nil},
+                                          "plan3"=>{:state=>"paused", :result=>nil},
+                                          "plan4"=>{:state=>"paused", :result=>nil}})
+
+              loaded_plans = adapter.find_execution_plan_statuses(filters: { label: ['test1'], :delayed => true })
+              _(loaded_plans).must_equal({})
+            end
+          end
+        end
+
         describe '#def find_execution_plan_counts' do
           before do
             # the tests expect clean field
@@ -331,6 +371,20 @@ module Dynflow
             _(plans.first[:execution_plan_uuid]).must_equal 'plan1'
           end
         end
+
+        describe '#delete_output_chunks' do
+          it 'deletes output chunks' do
+            prepare_plans_with_actions
+
+            adapter.save_output_chunks('plan1', 1, [{chunk: "Hello", timestamp: Time.now}, {chunk: "Bye", timestamp: Time.now}])
+            chunks = adapter.load_output_chunks('plan1', 1)
+            _(chunks.length).must_equal 2
+            deleted = adapter.delete_output_chunks('plan1', 1)
+            _(deleted).must_equal 2
+            chunks = adapter.load_output_chunks('plan1', 1)
+            _(chunks.length).must_equal 0
+          end
+        end
       end
 
       describe Dynflow::PersistenceAdapters::Sequel do
@@ -464,6 +518,27 @@ module Dynflow
           loaded_plan = adapter.load_execution_plan(plan[:id])
           assert_equal_attributes!(plan_data, loaded_plan)
         end
+
+        it 'does not leak Sequel blobs' do
+          db = adapter.send(:db)
+          # Prepare records for saving
+          plan = prepare_plans.first
+
+          value = 'a' * 1000
+
+          adata = action_data.merge({:output => { :key => value }})
+          plan_record = adapter.send(:prepare_record, :execution_plan, plan.merge(:uuid => plan[:id]))
+          action_record = adapter.send(:prepare_record, :action, adata.dup)
+
+          # Insert the records
+          db[:dynflow_execution_plans].insert plan_record.merge(:uuid => plan[:id])
+          db[:dynflow_actions].insert action_record.merge(:execution_plan_uuid => plan[:id], :id => adata[:id])
+
+          # Load the saved records
+          loaded_action = adapter.load_action(plan[:id], adata[:id])
+          _(loaded_action[:output][:key].class).must_equal String
+          _(loaded_action[:output][:key]).must_equal value
+        end
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dynflow
 version: !ruby/object:Gem::Version
-  version: 1.6.10
+  version: 1.7.0
 platform: ruby
 authors:
 - Ivan Necas
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-03-01 00:00:00.000000000 Z
+date: 2023-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -678,7 +678,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.20
+rubygems_version: 3.4.12
 signing_key:
 specification_version: 4
 summary: DYNamic workFLOW engine