dynflow 1.6.2 → 1.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f0e5c6a8141eaea583bebabd135cdbfcff6c2516410db7930595e3868383f331
-  data.tar.gz: bc72a36dd5284f5cea95651de2620f0d41db5f315195a4ed573155c94b510bb9
+  metadata.gz: 40e7ab36f1ef2943d1cbebccde747a2c89186a620493a74bfefb06a810e8bf13
+  data.tar.gz: c2d4da5a39382f4df50ac0a1664edab9b772f94fc6da4f7f5eec51c24b91e256
 SHA512:
-  metadata.gz: c9b0efe531cf9d3c45432bfc94542f47ff6a72fe20bd96a4541242f58a13abc77603a62fcb5294e481feb9a28c49c6eebd2a9747092da6fe7a5e67ad1dc8e9d3
-  data.tar.gz: d9c66e79fad07d6e6bfb9b8e265af0062adf8ba2b0d8d126e46080ca890200db07346f7c4702518a9c4d4ace4ae94d668ce20761166c9e81a7ef5531cdc85240
+  metadata.gz: 955818d3401e8641df7ad06ad0307791502437ac3b2549dee9d1161c8dd29d76e83f03b378b6c451547d2e1a00f0b8bbdfa282ea9b6e3fc37005e1fde59ebff6
+  data.tar.gz: 2cb868cab752e95928521d1276bca3091f3ebbd55437e80a82baf980c636880dea3bdb3b345252e2735ffe236960c9d9c6d78a471acc9462c8341581ec91ff98

data/.github/workflows/release.yml

@@ -0,0 +1,48 @@
+# workflow name
+name: Generate release-artifacts
+
+# on events
+on:
+  push:
+    tags:
+        - '*'
+
+# workflow tasks
+jobs:
+  generate:
+    name: Generate build artifacts
+    runs-on: ubuntu-latest
+    steps:
+      - uses: olegtarasov/get-tag@v2.1
+        id: tagName
+        with:
+          tagRegex: "v(.*)"  # Optional. Returns specified group text as tag name. Full tag string is returned if regex is not defined.
+          tagRegexGroup: 1 # Optional. Default is 1.
+      - name: Checkout the repository
+        uses: actions/checkout@v2
+      - name: Generate build files
+        run: |
+          mkdir -p dist
+          cd extras/expand
+          go build -o ../../dist/dynflow-expand-${VERSION}-x86_64
+        env:
+          VERSION: '${{ steps.tagName.outputs.tag }}'
+      - name: Generate distribution tarball
+        run: |
+          cd extras/expand
+          go mod vendor
+          tar --create \
+            --gzip \
+            --file ../../dist/dynflow-expand-${VERSION}.tar.gz \
+            --transform s/^\./dynflow-expand-${VERSION}/ \
+            .
+        env:
+          VERSION: '${{ steps.tagName.outputs.tag }}'
+      - name: Upload binaries to release
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: dist/*
+          tag: ${{ github.ref }}
+          overwrite: true
+          file_glob: true

data/Gemfile

@@ -35,7 +35,7 @@ end
 
 group :rails do
   gem 'daemons'
-  gem 'rails', '>= 4.2.9'
+  gem 'rails', '>= 4.2.9', '< 7'
   gem 'logging'
 end
 

data/extras/expand/Dockerfile

@@ -0,0 +1,9 @@
+FROM alpine:3.15 as builder
+RUN apk add -U go
+ADD ./ work/
+RUN cd /work && \
+    go build
+
+FROM scratch
+COPY --from=builder /work/expand /expand
+CMD ["/expand"]

data/extras/expand/README.md

@@ -0,0 +1,25 @@
+# expand
+
+For a long time, Dynflow's database schema remained stable. To optimize Dynflow
+a bit, we started changing it. One of the changes was changing how we encode
+flows, resulting in flows taking roughly 10x less space.
+
+The other change is not merged yet, but has potentionally bigger impact. We
+store certain columns as JSON objects. The upcoming change uses msgpack instead
+of JSON, resulting in faster encoding and decoding times and smaller storage
+footprint when encoded. The drawback is it is a binary format, so if someone
+dumps the tables from DB as CSV, they won't be human readable.
+
+This tool processes CSV DB dumps and decodes msgpack to json.
+
+## Usage
+
+```shell
+# cat dynflow_execution_plans.csv
+2065cc55-6b03-44b7-947a-e999dcb9057f,,stopped,error,,2021-04-16 09:50:33.826,0,0,,Dynflow::ExecutionPlan,1,\x91a143,\x91a153,\x9283a474696d65ce60795de9a46e616d65a564656c6179a8776f726c645f6964d92435626536643435662d363732342d343666652d393035662d34363565316466346561306183a474696d65ce60795de9a46e616d65a774696d656f7574a8776f726c645f6964d92435626536643435662d363732342d343666652d393035662d343635653164663465613061,\x9101
+6667374a-beab-4b0b-80c8-3d0392cdde40,,scheduled,pending,,,0,,,Dynflow::ExecutionPlan,1,\x91a143,\x91a153,\x9183a474696d65ce60795de9a46e616d65a564656c6179a8776f726c645f6964d92435626536643435662d363732342d343666652d393035662d343635653164663465613061,\x9101
+
+# expand < dynflow_execution_plans.csv
+2065cc55-6b03-44b7-947a-e999dcb9057f,,stopped,error,,2021-04-16 09:50:33.826,0,0,,Dynflow::ExecutionPlan,1,"[""C""]","[""S""]","[{""name"":""delay"",""time"":1618566633,""world_id"":""5be6d45f-6724-46fe-905f-465e1df4ea0a""},{""name"":""timeout"",""time"":1618566633,""world_id"":""5be6d45f-6724-46fe-905f-465e1df4ea0a""}]",[1]
+6667374a-beab-4b0b-80c8-3d0392cdde40,,scheduled,pending,,,0,,,Dynflow::ExecutionPlan,1,"[""C""]","[""S""]","[{""name"":""delay"",""time"":1618566633,""world_id"":""5be6d45f-6724-46fe-905f-465e1df4ea0a""}]",[1]
+```

data/extras/expand/go.mod

@@ -0,0 +1,5 @@
+module github.com/dynflow/dynflow/expand
+
+go 1.15
+
+require github.com/vmihailenco/msgpack/v5 v5.3.5

data/extras/expand/go.sum

@@ -0,0 +1,11 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/vmihailenco/msgpack v3.3.3+incompatible h1:wapg9xDUZDzGCNFlwc5SqI1rvcciqcxEHac4CYj89xI=
+github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=
+github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
+github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
+github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

data/extras/expand/main.go

@@ -4,47 +4,17 @@ import (
 	"encoding/csv"
 	"encoding/hex"
 	"encoding/json"
-	"github.com/vmihailenco/msgpack"
+	"github.com/vmihailenco/msgpack/v5"
 	"io"
 	"os"
 )
 
-// dynflow_steps
-// 0                   1  2         3    4     5          6        7         8              9             10              11    12    13           14       15
-// execution_plan_uuid,id,action_id,data,state,started_at,ended_at,real_time,execution_time,progress_done,progress_weight,class,error,action_class,children,queue
-//
-// encoded columns are:
-// 3 - data
-// 12 - error
-// 14 - children
-
-// dynflow_actions
-// 0                   1  2    3                        4                5     6     7      8            9           10
-// execution_plan_uuid,id,data,caller_execution_plan_id,caller_action_id,class,input,output,plan_step_id,run_step_id,finalize_step_id
-//
-// encoded columns are:
-// 2 - data
-// 6 - input
-// 7 - output
-
-// dynflow_execution_plans
-// Without msgpack
-// 0    1    2     3      4          5        6         7              8     9     10       11            12                13                14
-// uuid,data,state,result,started_at,ended_at,real_time,execution_time,label,class,run_flow,finalize_flow,execution_history,root_plan_step_id,step_ids
-
-// With msgpack
-// 0    1    2     3      4          5        6         7              8     9     10                11       12            13                14
-// uuid,data,state,result,started_at,ended_at,real_time,execution_time,label,class,root_plan_step_id,run_flow,finalize_flow,execution_history,step_ids
-//
-// 1 - data
-// 11 - run_flow
-// 12 - finalize_flow
-// 13 - execution_history
-// 14 - step_ids
-
 func main() {
 	reader := csv.NewReader(os.Stdin)
+	defer os.Stdin.Close()
+
 	writer := csv.NewWriter(os.Stdout)
+	defer os.Stdout.Close()
 	defer writer.Flush()
 
 	for {
@@ -58,31 +28,12 @@ func main() {
 }
 
 func processRow(record []string) []string {
-	// Execution plan exports have 15 fields, other exports have different counts
-	if len(record) == 15 {
-		record = expandExecutionPlan(record)
-	}
-
 	for i, r := range record {
-		record[i] = reencodeField(r)
-	}
-
-	return record
-}
-
-func expandExecutionPlan(record []string) []string {
-	var flow_columns [2]int
-
-	// The step_ids field should be a safe indicator
-	if isHexEncoded(record[14]) {
-		flow_columns = [...]int{11, 12}
-	} else {
-		flow_columns = [...]int{10, 11}
+		if isHexEncoded(r) {
+			record[i] = reencodeField(r)
+		}
 	}
 
-	for _, i := range flow_columns {
-		record[i] = expandFlow(record[i])
-	}
 	return record
 }
 
@@ -91,38 +42,18 @@ func isHexEncoded(field string) bool {
 }
 
 func reencodeField(field string) string {
-	decoded, err := decode(field)
+	decoded_bytes, err := hex.DecodeString(field[2:])
 	if err != nil {
 		return field
 	}
 
-	return encode(decoded)
-}
-
-func decode(field string) (interface{}, error) {
 	var intermediate interface{}
-	bytes := []byte(field)
-
-	if isHexEncoded(field) {
-		decoded_bytes, err := hex.DecodeString(field[2:])
-		if err != nil {
-			return "", err
-		}
-
-		err = msgpack.Unmarshal(decoded_bytes, &intermediate)
-		if err != nil {
-			return "", err
-		}
-
-		return intermediate, nil
-	}
-
-	err := json.Unmarshal(bytes, &intermediate)
+	err = msgpack.Unmarshal(decoded_bytes, &intermediate)
 	if err != nil {
-		return "", err
+		return field
 	}
 
-	return intermediate, nil
+	return encode(intermediate)
 }
 
 func encode(data interface{}) string {
@@ -133,48 +64,3 @@ func encode(data interface{}) string {
 
 	return string(result)
 }
-
-func expandFlow(field string) string {
-	intermediate, err := decode(field)
-	if err != nil {
-		return field
-	}
-
-	var result map[string]interface{}
-	switch intermediate.(type) {
-	// old style hash
-	case map[string]interface{}:
-		result = intermediate.(map[string]interface{})
-	// newer compact S-expression like representation
-	case []interface{}, float64:
-		result = expandCompactFlow(intermediate)
-	}
-
-	return encode(result)
-}
-
-func expandCompactFlow(flow interface{}) map[string]interface{} {
-	result := make(map[string]interface{})
-	switch flow.(type) {
-	case []interface{}:
-		switch flow.([]interface{})[0] {
-		case "S":
-			result["class"] = "Dynflow::Flows::Sequence"
-		case "C":
-			result["class"] = "Dynflow::Flows::Concurrence"
-		default:
-			panic("Unknown flow type")
-		}
-		var subflows []interface{}
-		for subflow := range flow.([]interface{})[1:] {
-			subflows = append(subflows, expandCompactFlow(subflow))
-		}
-		result["flows"] = subflows
-	case float64, int:
-		result["class"] = "Dynflow::Flows::Atom"
-		result["step_id"] = flow
-	default:
-		panic("Unknown flow type")
-	}
-	return result
-}

data/lib/dynflow/persistence_adapters/sequel_migrations/022_store_flows_as_msgpack.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'multi_json'
+require 'json'
 require 'msgpack'
 
 def table_pkeys(table)
@@ -74,7 +74,7 @@ Sequel.migration do
       new_columns = columns.map { |c| "#{c}_blob" }
 
       migrate_table table, columns, new_columns, File do |data|
-        ::Sequel.blob(MessagePack.pack(MultiJson.load(data)))
+        ::Sequel.blob(MessagePack.pack(JSON.parse(data)))
       end
     end
   end
@@ -83,7 +83,7 @@ Sequel.migration do
     TABLES.each do |table, columns|
       new_columns = columns.map { |c| c + '_text' }
       migrate_table table, columns, new_columns, String do |data|
-        MultiJson.dump(MessagePack.unpack(data))
+        JSON.dump(MessagePack.unpack(data))
       end
     end
   end

data/lib/dynflow/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Dynflow
-  VERSION = '1.6.2'
+  VERSION = '1.6.3'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dynflow
 version: !ruby/object:Gem::Version
-  version: 1.6.2
+  version: 1.6.3
 platform: ruby
 authors:
 - Ivan Necas
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-10 00:00:00.000000000 Z
+date: 2022-01-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multi_json
@@ -263,6 +263,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".github/install_dependencies.sh"
+- ".github/workflows/release.yml"
 - ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rubocop.yml"
@@ -419,6 +420,10 @@ files:
 - examples/sub_plan_concurrency_control.rb
 - examples/sub_plans.rb
 - examples/termination.rb
+- extras/expand/Dockerfile
+- extras/expand/README.md
+- extras/expand/go.mod
+- extras/expand/go.sum
 - extras/expand/main.go
 - extras/statsd_mapping.conf
 - lib/dynflow.rb