dynflow 1.6.1 → 1.6.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f660ffddfd3e3c7ea4b8414d2e6f27f48d897d34280ff3bec3c8ab6e360b0bfe
-  data.tar.gz: 46cd643c84f09640a07f00eafc3f4d3feda65746d3c353338d8141277fd77e03
+  metadata.gz: '068120950f509e0e118319244c51c2cc1c593b95c808e550636515dace4a984b'
+  data.tar.gz: 0a7b42cc5c57ba401c983582cebd1b428c6a9876d6187f3e511bcf5b82f33d4e
 SHA512:
-  metadata.gz: 99e59816fbf809403a87bbb1f96ff39958d3d059572de7a66f6205df7c48867baa50d77636d287aebb8c69d73db2cef686860520c6e7db27dbfa8217f5570715
-  data.tar.gz: 315b37f302e1c29eae52d00acabd82b605d88b7982ba3e01c920dd18619967115875ec5a1facffc6299280f01aaa61db8162ce3e02bbbd0ba06a95add1b88d64
+  metadata.gz: fa1c11576f57765c44d2efb27d80237e07e07733da768e79d2622cce9e5cdea295ae18894601905a431632c3a261668bb65a851f340baec681bbabec3e2d7dc1
+  data.tar.gz: 51d939e678a1e90880767b772d10f25fb69870da708180d9c07a42e46840428a929620206a5a05f1014c17bc7e5d7f71d8eff92e0c99654e1ec31a9ab73d2889

data/.github/workflows/release.yml

@@ -0,0 +1,48 @@
+# workflow name
+name: Generate release-artifacts
+
+# on events
+on:
+  push:
+    tags:
+        - '*'
+
+# workflow tasks
+jobs:
+  generate:
+    name: Generate build artifacts
+    runs-on: ubuntu-latest
+    steps:
+      - uses: olegtarasov/get-tag@v2.1
+        id: tagName
+        with:
+          tagRegex: "v(.*)"  # Optional. Returns specified group text as tag name. Full tag string is returned if regex is not defined.
+          tagRegexGroup: 1 # Optional. Default is 1.
+      - name: Checkout the repository
+        uses: actions/checkout@v2
+      - name: Generate build files
+        run: |
+          mkdir -p dist
+          cd extras/expand
+          go build -o ../../dist/dynflow-expand-${VERSION}-x86_64
+        env:
+          VERSION: '${{ steps.tagName.outputs.tag }}'
+      - name: Generate distribution tarball
+        run: |
+          cd extras/expand
+          go mod vendor
+          tar --create \
+            --gzip \
+            --file ../../dist/dynflow-expand-${VERSION}.tar.gz \
+            --transform s/^\./dynflow-expand-${VERSION}/ \
+            .
+        env:
+          VERSION: '${{ steps.tagName.outputs.tag }}'
+      - name: Upload binaries to release
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: dist/*
+          tag: ${{ github.ref }}
+          overwrite: true
+          file_glob: true

data/Gemfile

@@ -35,7 +35,7 @@ end
 
 group :rails do
   gem 'daemons'
-  gem 'rails', '>= 4.2.9'
+  gem 'rails', '>= 4.2.9', '< 7'
   gem 'logging'
 end
 

data/dynflow.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
   s.required_ruby_version = '>= 2.3.0'
 
   s.add_dependency "multi_json"
-  s.add_dependency "msgpack", '~> 1.3.3'
+  s.add_dependency "msgpack", '~> 1.3', '>= 1.3.3'
   s.add_dependency "apipie-params"
   s.add_dependency "algebrick", '~> 0.7.0'
   s.add_dependency "concurrent-ruby", '~> 1.1.3'

data/extras/expand/Dockerfile

@@ -0,0 +1,9 @@
+FROM alpine:3.15 as builder
+RUN apk add -U go
+ADD ./ work/
+RUN cd /work && \
+    go build
+
+FROM scratch
+COPY --from=builder /work/expand /expand
+CMD ["/expand"]

data/extras/expand/README.md

@@ -0,0 +1,25 @@
+# expand
+
+For a long time, Dynflow's database schema remained stable. To optimize Dynflow
+a bit, we started changing it. One of the changes was changing how we encode
+flows, resulting in flows taking roughly 10x less space.
+
+The other change is not merged yet, but has potentionally bigger impact. We
+store certain columns as JSON objects. The upcoming change uses msgpack instead
+of JSON, resulting in faster encoding and decoding times and smaller storage
+footprint when encoded. The drawback is it is a binary format, so if someone
+dumps the tables from DB as CSV, they won't be human readable.
+
+This tool processes CSV DB dumps and decodes msgpack to json.
+
+## Usage
+
+```shell
+# cat dynflow_execution_plans.csv
+2065cc55-6b03-44b7-947a-e999dcb9057f,,stopped,error,,2021-04-16 09:50:33.826,0,0,,Dynflow::ExecutionPlan,1,\x91a143,\x91a153,\x9283a474696d65ce60795de9a46e616d65a564656c6179a8776f726c645f6964d92435626536643435662d363732342d343666652d393035662d34363565316466346561306183a474696d65ce60795de9a46e616d65a774696d656f7574a8776f726c645f6964d92435626536643435662d363732342d343666652d393035662d343635653164663465613061,\x9101
+6667374a-beab-4b0b-80c8-3d0392cdde40,,scheduled,pending,,,0,,,Dynflow::ExecutionPlan,1,\x91a143,\x91a153,\x9183a474696d65ce60795de9a46e616d65a564656c6179a8776f726c645f6964d92435626536643435662d363732342d343666652d393035662d343635653164663465613061,\x9101
+
+# expand < dynflow_execution_plans.csv
+2065cc55-6b03-44b7-947a-e999dcb9057f,,stopped,error,,2021-04-16 09:50:33.826,0,0,,Dynflow::ExecutionPlan,1,"[""C""]","[""S""]","[{""name"":""delay"",""time"":1618566633,""world_id"":""5be6d45f-6724-46fe-905f-465e1df4ea0a""},{""name"":""timeout"",""time"":1618566633,""world_id"":""5be6d45f-6724-46fe-905f-465e1df4ea0a""}]",[1]
+6667374a-beab-4b0b-80c8-3d0392cdde40,,scheduled,pending,,,0,,,Dynflow::ExecutionPlan,1,"[""C""]","[""S""]","[{""name"":""delay"",""time"":1618566633,""world_id"":""5be6d45f-6724-46fe-905f-465e1df4ea0a""}]",[1]
+```

data/extras/expand/go.mod

@@ -0,0 +1,5 @@
+module github.com/dynflow/dynflow/expand
+
+go 1.15
+
+require github.com/vmihailenco/msgpack/v5 v5.3.5

data/extras/expand/go.sum

@@ -0,0 +1,11 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/vmihailenco/msgpack v3.3.3+incompatible h1:wapg9xDUZDzGCNFlwc5SqI1rvcciqcxEHac4CYj89xI=
+github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=
+github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
+github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
+github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

data/extras/expand/main.go

@@ -4,47 +4,17 @@ import (
 	"encoding/csv"
 	"encoding/hex"
 	"encoding/json"
-	"github.com/vmihailenco/msgpack"
+	"github.com/vmihailenco/msgpack/v5"
 	"io"
 	"os"
 )
 
-// dynflow_steps
-// 0                   1  2         3    4     5          6        7         8              9             10              11    12    13           14       15
-// execution_plan_uuid,id,action_id,data,state,started_at,ended_at,real_time,execution_time,progress_done,progress_weight,class,error,action_class,children,queue
-//
-// encoded columns are:
-// 3 - data
-// 12 - error
-// 14 - children
-
-// dynflow_actions
-// 0                   1  2    3                        4                5     6     7      8            9           10
-// execution_plan_uuid,id,data,caller_execution_plan_id,caller_action_id,class,input,output,plan_step_id,run_step_id,finalize_step_id
-//
-// encoded columns are:
-// 2 - data
-// 6 - input
-// 7 - output
-
-// dynflow_execution_plans
-// Without msgpack
-// 0    1    2     3      4          5        6         7              8     9     10       11            12                13                14
-// uuid,data,state,result,started_at,ended_at,real_time,execution_time,label,class,run_flow,finalize_flow,execution_history,root_plan_step_id,step_ids
-
-// With msgpack
-// 0    1    2     3      4          5        6         7              8     9     10                11       12            13                14
-// uuid,data,state,result,started_at,ended_at,real_time,execution_time,label,class,root_plan_step_id,run_flow,finalize_flow,execution_history,step_ids
-//
-// 1 - data
-// 11 - run_flow
-// 12 - finalize_flow
-// 13 - execution_history
-// 14 - step_ids
-
 func main() {
 	reader := csv.NewReader(os.Stdin)
+	defer os.Stdin.Close()
+
 	writer := csv.NewWriter(os.Stdout)
+	defer os.Stdout.Close()
 	defer writer.Flush()
 
 	for {
@@ -58,31 +28,12 @@ func main() {
 }
 
 func processRow(record []string) []string {
-	// Execution plan exports have 15 fields, other exports have different counts
-	if len(record) == 15 {
-		record = expandExecutionPlan(record)
-	}
-
 	for i, r := range record {
-		record[i] = reencodeField(r)
-	}
-
-	return record
-}
-
-func expandExecutionPlan(record []string) []string {
-	var flow_columns [2]int
-
-	// The step_ids field should be a safe indicator
-	if isHexEncoded(record[14]) {
-		flow_columns = [...]int{11, 12}
-	} else {
-		flow_columns = [...]int{10, 11}
+		if isHexEncoded(r) {
+			record[i] = reencodeField(r)
+		}
 	}
 
-	for _, i := range flow_columns {
-		record[i] = expandFlow(record[i])
-	}
 	return record
 }
 
@@ -91,38 +42,18 @@ func isHexEncoded(field string) bool {
 }
 
 func reencodeField(field string) string {
-	decoded, err := decode(field)
+	decoded_bytes, err := hex.DecodeString(field[2:])
 	if err != nil {
 		return field
 	}
 
-	return encode(decoded)
-}
-
-func decode(field string) (interface{}, error) {
 	var intermediate interface{}
-	bytes := []byte(field)
-
-	if isHexEncoded(field) {
-		decoded_bytes, err := hex.DecodeString(field[2:])
-		if err != nil {
-			return "", err
-		}
-
-		err = msgpack.Unmarshal(decoded_bytes, &intermediate)
-		if err != nil {
-			return "", err
-		}
-
-		return intermediate, nil
-	}
-
-	err := json.Unmarshal(bytes, &intermediate)
+	err = msgpack.Unmarshal(decoded_bytes, &intermediate)
 	if err != nil {
-		return "", err
+		return field
 	}
 
-	return intermediate, nil
+	return encode(intermediate)
 }
 
 func encode(data interface{}) string {
@@ -133,48 +64,3 @@ func encode(data interface{}) string {
 
 	return string(result)
 }
-
-func expandFlow(field string) string {
-	intermediate, err := decode(field)
-	if err != nil {
-		return field
-	}
-
-	var result map[string]interface{}
-	switch intermediate.(type) {
-	// old style hash
-	case map[string]interface{}:
-		result = intermediate.(map[string]interface{})
-	// newer compact S-expression like representation
-	case []interface{}, float64:
-		result = expandCompactFlow(intermediate)
-	}
-
-	return encode(result)
-}
-
-func expandCompactFlow(flow interface{}) map[string]interface{} {
-	result := make(map[string]interface{})
-	switch flow.(type) {
-	case []interface{}:
-		switch flow.([]interface{})[0] {
-		case "S":
-			result["class"] = "Dynflow::Flows::Sequence"
-		case "C":
-			result["class"] = "Dynflow::Flows::Concurrence"
-		default:
-			panic("Unknown flow type")
-		}
-		var subflows []interface{}
-		for subflow := range flow.([]interface{})[1:] {
-			subflows = append(subflows, expandCompactFlow(subflow))
-		}
-		result["flows"] = subflows
-	case float64, int:
-		result["class"] = "Dynflow::Flows::Atom"
-		result["step_id"] = flow
-	default:
-		panic("Unknown flow type")
-	}
-	return result
-}

data/lib/dynflow/execution_plan/hooks.rb

@@ -69,7 +69,7 @@ module Dynflow
                 action.send(hook, execution_plan)
               rescue => e
                 execution_plan.logger.error "Failed to run hook '#{hook}' for action '#{action.class}'"
-                execution_plan.logger.debug e
+                execution_plan.logger.error e
               end
             end
           end

data/lib/dynflow/persistence_adapters/sequel_migrations/022_store_flows_as_msgpack.rb

@@ -1,65 +1,9 @@
 # frozen_string_literal: true
 
-require 'multi_json'
-require 'msgpack'
-
-def table_pkeys(table)
-  case table
-  when :dynflow_execution_plans
-    [:uuid]
-  when :dynflow_actions, :dynflow_steps
-    [:execution_plan_uuid, :id]
-  when :dynflow_coordinator_records
-    [:id, :class]
-  when :dynflow_delayed_plans
-    [:execution_plan_uuid]
-  when :dynflow_envelopes
-    [:id]
-  when :dynflow_output_chunks
-    [:chunk]
-  else
-    raise "Unknown table '#{table}'"
-  end
-end
-
-def conditions_for_row(table, row)
-  row.slice(*table_pkeys(table))
-end
-
-def migrate_table(table, from_names, to_names, new_type)
-  alter_table(table) do
-    to_names.each do |new|
-      add_column new, new_type
-    end
-  end
-
-  relevant_columns = table_pkeys(table) | from_names
-
-  from(table).select(*relevant_columns).each do |row|
-    update = from_names.zip(to_names).reduce({}) do |acc, (from, to)|
-      row[from].nil? ? acc : acc.merge(to => yield(row[from]))
-    end
-    next if update.empty?
-    from(table).where(conditions_for_row(table, row)).update(update)
-  end
-
-  from_names.zip(to_names).each do |old, new|
-    alter_table(table) do
-      drop_column old
-    end
-
-    if database_type == :mysql
-      type = new_type == File ? 'blob' : 'mediumtext'
-      run "ALTER TABLE #{table} CHANGE COLUMN `#{new}` `#{old}` #{type};"
-    else
-      rename_column table, new, old
-    end
-  end
-end
+require_relative 'msgpack_migration_helper'
 
 Sequel.migration do
-
-  TABLES = {
+  helper = MsgpackMigrationHelper.new({
     :dynflow_actions => [:data, :input, :output],
     :dynflow_coordinator_records => [:data],
     :dynflow_delayed_plans => [:serialized_args, :data],
@@ -67,24 +11,13 @@ Sequel.migration do
     :dynflow_execution_plans => [:run_flow, :finalize_flow, :execution_history, :step_ids],
     :dynflow_steps => [:error, :children],
     :dynflow_output_chunks => [:chunk]
-  }
+  })
 
   up do
-    TABLES.each do |table, columns|
-      new_columns = columns.map { |c| "#{c}_blob" }
-
-      migrate_table table, columns, new_columns, File do |data|
-        ::Sequel.blob(MessagePack.pack(MultiJson.load(data)))
-      end
-    end
+    helper.up(self)
   end
 
   down do
-    TABLES.each do |table, columns|
-      new_columns = columns.map { |c| c + '_text' }
-      migrate_table table, columns, new_columns, String do |data|
-        MultiJson.dump(MessagePack.unpack(data))
-      end
-    end
+    helper.down(self)
   end
 end

data/lib/dynflow/persistence_adapters/sequel_migrations/024_store_execution_plan_data_as_msgpack.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require_relative 'msgpack_migration_helper'
+
+Sequel.migration do
+  helper = MsgpackMigrationHelper.new({
+    :dynflow_execution_plans => [:data],
+    :dynflow_steps => [:data]
+  })
+
+  up do
+    helper.up(self)
+  end
+
+  down do
+    helper.down(self)
+  end
+end

data/lib/dynflow/persistence_adapters/sequel_migrations/msgpack_migration_helper.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'msgpack'
+
+class MsgpackMigrationHelper
+  def initialize(tables)
+    @tables = tables
+  end
+
+  def up(migration)
+    @tables.each do |table, columns|
+      new_columns = columns.map { |c| "#{c}_blob" }
+
+      migrate_table migration, table, columns, new_columns, File do |data|
+        ::Sequel.blob(MessagePack.pack(JSON.parse(data)))
+      end
+    end
+  end
+
+  def down(migration)
+    @tables.each do |table, columns|
+      new_columns = columns.map { |c| c + '_text' }
+      migrate_table migration, table, columns, new_columns, String do |data|
+        JSON.dump(MessagePack.unpack(data))
+      end
+    end
+  end
+
+  private
+
+  def migrate_table(migration, table, from_names, to_names, new_type)
+    migration.alter_table(table) do
+      to_names.each do |new|
+        add_column new, new_type
+      end
+    end
+
+    relevant_columns = table_pkeys(table) | from_names
+
+    migration.from(table).select(*relevant_columns).each do |row|
+      update = from_names.zip(to_names).reduce({}) do |acc, (from, to)|
+        row[from].nil? ? acc : acc.merge(to => yield(row[from]))
+      end
+      next if update.empty?
+      migration.from(table).where(conditions_for_row(table, row)).update(update)
+    end
+
+    from_names.zip(to_names).each do |old, new|
+      migration.alter_table(table) do
+        drop_column old
+      end
+
+      if migration.database_type == :mysql
+        type = new_type == File ? 'blob' : 'mediumtext'
+        run "ALTER TABLE #{table} CHANGE COLUMN `#{new}` `#{old}` #{type};"
+      else
+        migration.rename_column table, new, old
+      end
+    end
+  end
+
+  def conditions_for_row(table, row)
+    row.slice(*table_pkeys(table))
+  end
+
+  def table_pkeys(table)
+    case table
+    when :dynflow_execution_plans
+      [:uuid]
+    when :dynflow_actions, :dynflow_steps
+      [:execution_plan_uuid, :id]
+    when :dynflow_coordinator_records
+      [:id, :class]
+    when :dynflow_delayed_plans
+      [:execution_plan_uuid]
+    when :dynflow_envelopes
+      [:id]
+    when :dynflow_output_chunks
+      [:id]
+    else
+      raise "Unknown table '#{table}'"
+    end
+  end
+end

data/lib/dynflow/rails/configuration.rb

@@ -158,7 +158,7 @@ module Dynflow
       protected
 
       def default_sequel_adapter_options(world)
-        db_config            = ::ActiveRecord::Base.configurations[::Rails.env].dup
+        db_config            = ::Rails.application.config.database_configuration[::Rails.env].dup
         db_config['adapter'] = db_config['adapter'].gsub(/_?makara_?/, '')
         db_config['adapter'] = 'postgres' if db_config['adapter'] == 'postgresql'
         db_config['max_connections'] = calculate_db_pool_size(world) if increase_db_pool_size?

data/lib/dynflow/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Dynflow
-  VERSION = '1.6.1'
+  VERSION = '1.6.4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dynflow
 version: !ruby/object:Gem::Version
-  version: 1.6.1
+  version: 1.6.4
 platform: ruby
 authors:
 - Ivan Necas
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-09-24 00:00:00.000000000 Z
+date: 2022-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -30,6 +30,9 @@ dependencies:
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.3.3
   type: :runtime
@@ -37,6 +40,9 @@ dependencies:
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.3.3
 - !ruby/object:Gem::Dependency
@@ -257,6 +263,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".github/install_dependencies.sh"
+- ".github/workflows/release.yml"
 - ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rubocop.yml"
@@ -413,6 +420,10 @@ files:
 - examples/sub_plan_concurrency_control.rb
 - examples/sub_plans.rb
 - examples/termination.rb
+- extras/expand/Dockerfile
+- extras/expand/README.md
+- extras/expand/go.mod
+- extras/expand/go.sum
 - extras/expand/main.go
 - extras/statsd_mapping.conf
 - lib/dynflow.rb
@@ -536,6 +547,8 @@ files:
 - lib/dynflow/persistence_adapters/sequel_migrations/021_create_output_chunks.rb
 - lib/dynflow/persistence_adapters/sequel_migrations/022_store_flows_as_msgpack.rb
 - lib/dynflow/persistence_adapters/sequel_migrations/023_sqlite_workarounds.rb
+- lib/dynflow/persistence_adapters/sequel_migrations/024_store_execution_plan_data_as_msgpack.rb
+- lib/dynflow/persistence_adapters/sequel_migrations/msgpack_migration_helper.rb
 - lib/dynflow/rails.rb
 - lib/dynflow/rails/configuration.rb
 - lib/dynflow/rails/daemon.rb