mushy 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 4983e312423619f6384adcf954591ebc267020291d05ded5ac5b4f4c3a5e66f5
4
+ data.tar.gz: ff7265cafa27a7ef94be0e4c99c7e7be0f46448a32efc5b66fa4e6c25739378e
5
+ SHA512:
6
+ metadata.gz: 6490c4b94d2de183f948ca3cdf70f30e7000ed191b061f427aba15e367c665428cb3aac9af4a95b892c1de01672bef1709e47a48bda41b4391864f2a0af6c35f
7
+ data.tar.gz: 53b0e05775e441c657711106858fc38a03709ffc36a0bd5430301c2a067aa785d931439f2e2d3d9b46a267d02e841afcbb5128800bbf77ff6dd54bd51a97cbd2
@@ -0,0 +1,3 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ puts 'mushy'
@@ -0,0 +1,15 @@
1
+ require 'liquid'
2
+ require 'symbolized'
3
+
4
+ Dir[File.dirname(__FILE__) + '/mushy/*.rb'].each { |f| require f }
5
+
6
+ important_flux_files = ['bash'].map { |x| "#{x}.rb" }
7
+ Dir[File.dirname(__FILE__) + '/mushy/fluxs/*.rb']
8
+ .sort_by { |f| important_flux_files.any? { |x| f.end_with?(x) } ? 0 : 1 }
9
+ .each { |f| require f }
10
+
11
+ module Mushy
12
+ def self.hi
13
+ puts 'hello'
14
+ end
15
+ end
@@ -0,0 +1,15 @@
1
+ module Mushy
2
+
3
+ class Event
4
+ attr_accessor :id
5
+ attr_accessor :run_id
6
+ attr_accessor :flux_id
7
+ attr_accessor :flow_id
8
+ attr_accessor :data
9
+
10
+ def initialize
11
+ self.data = SymbolizedHash.new
12
+ end
13
+ end
14
+
15
+ end
@@ -0,0 +1,47 @@
1
+ require 'json'
2
+
3
+ module Mushy
4
+
5
+ class Flow
6
+ attr_accessor :id
7
+ attr_accessor :fluxs
8
+
9
+ def initialize
10
+ self.id = SecureRandom.uuid
11
+ self.fluxs = []
12
+ end
13
+
14
+ def fluxs_for event
15
+ fluxs
16
+ .select { |x| x.parent_fluxs.any? { |y| y.id == event.flux_id } }
17
+ .flatten
18
+ end
19
+
20
+ def self.build_flux record
21
+ flux = Object.const_get("Mushy::#{record[:type] || record['type'] || 'Flux'}").new
22
+ flux.id = record[:id] || record['id'] || flux.id
23
+ flux.config = SymbolizedHash.new(record[:config] || record['config'])
24
+ flux
25
+ end
26
+
27
+ def self.parse data
28
+ data = JSON.parse data
29
+ flow = new
30
+
31
+ data_fluxs = data['fluxs'] || []
32
+
33
+ flow.fluxs = data_fluxs.map { |s| build_flux s }
34
+
35
+ fluxs_with_parent_ids = flow.fluxs.reduce({}) { |t, i| t[i.id] = []; t }
36
+ data_fluxs.map { |r| fluxs_with_parent_ids[r['id']] = r['parent_fluxs'] || [] }
37
+
38
+ flow.fluxs.each do |flux|
39
+ flux.parent_fluxs = flow.fluxs.select { |x| fluxs_with_parent_ids[flux.id].include?(x.id) }
40
+ end
41
+
42
+ flow
43
+ end
44
+
45
+ end
46
+
47
+ end
@@ -0,0 +1,165 @@
1
+ module Mushy
2
+
3
+ class Flux
4
+
5
+ attr_accessor :id
6
+ attr_accessor :parent_fluxs
7
+ attr_accessor :subscribed_to
8
+ attr_accessor :config
9
+ attr_accessor :masher
10
+
11
+ def initialize
12
+ guard
13
+ end
14
+
15
+ class << self
16
+ attr_accessor :all
17
+
18
+ def inherited subclass
19
+ if (self != Mushy::Flux)
20
+ Mushy::Flux.inherited subclass
21
+ else
22
+ self.all ||= []
23
+ self.all << subclass
24
+ end
25
+ end
26
+ end
27
+
28
+ def guard
29
+ self.id ||= SecureRandom.uuid
30
+ self.parent_fluxs ||= []
31
+ self.subscribed_to ||= []
32
+ self.masher ||= Masher.new
33
+ self.config ||= SymbolizedHash.new
34
+ end
35
+
36
+ def execute incoming_event
37
+ guard
38
+
39
+ incoming_event = SymbolizedHash.new(incoming_event) if incoming_event.is_a?(Hash)
40
+
41
+ event = incoming_event
42
+
43
+ mashed_config = masher.mash config, event
44
+
45
+ incoming_split = mashed_config[:incoming_split]
46
+
47
+ events = incoming_split ? incoming_event[incoming_split] : [event]
48
+
49
+ results = events.map { |e| execute_single_event e, config }
50
+
51
+ return results.first unless incoming_split
52
+
53
+ results = join_these_results(results, event, config[:join]) if config[:join]
54
+
55
+ results.flatten
56
+ end
57
+
58
+ def execute_single_event event, config
59
+
60
+ mashed_config = masher.mash config, event
61
+
62
+ the_original_join = mashed_config[:join]
63
+ mashed_config[:join] = nil if mashed_config[:incoming_split]
64
+
65
+ results = process event, mashed_config
66
+
67
+ returned_one_result = results.is_a?(Hash)
68
+
69
+ results = standardize_these results
70
+ results = shape_these results, event, mashed_config
71
+
72
+ return results.first if the_original_join
73
+
74
+ return results if mashed_config[:outgoing_split]
75
+
76
+ returned_one_result ? results.first : results
77
+
78
+ end
79
+
80
+ def standardize_these results
81
+ [results]
82
+ .flatten
83
+ .map { |x| x.is_a?(Hash) ? convert_to_symbolized_hash(x) : nil }
84
+ .select { |x| x }
85
+ end
86
+
87
+ def shape_these results, event, config
88
+ supported_shaping = [:merge, :outgoing_split, :group, :model, :join, :sort, :limit]
89
+
90
+ shaping = supported_shaping
91
+ if (config[:shaping])
92
+ shaping = convert_this_to_an_array(config[:shaping]).map { |x| x.to_sym }
93
+ end
94
+
95
+ supported_shaping
96
+ .select { |x| config[x] }
97
+ .each_with_index
98
+ .sort_by { |x, i| shaping.index(x) || i + supported_shaping.count }
99
+ .map { |x, _| x }
100
+ .reduce(results) { |t, i| self.send("#{i}_these_results".to_sym, t, event, config[i]) }
101
+ end
102
+
103
+ def sort_these_results results, event, by
104
+ results.sort { |x| x[by].to_i }
105
+ end
106
+
107
+ def limit_these_results results, event, by
108
+ results
109
+ .each_with_index
110
+ .select { |x, i| i < by.to_i }
111
+ .map { |x, _| x }
112
+ end
113
+
114
+ def group_these_results results, event, by
115
+ group_by = by.split('|')[0]
116
+ result_key = by.split('|')[1]
117
+ results.group_by { |x| x[group_by] }.map { |k, v| SymbolizedHash.new( { result_key => v } ) }
118
+ end
119
+
120
+ def outgoing_split_these_results results, event, by
121
+ results.map { |x| Masher.new.dig by, x }.flatten
122
+ end
123
+
124
+ def join_these_results results, event, by
125
+ [SymbolizedHash.new( { by => results } )]
126
+ end
127
+
128
+ def model_these_results results, event, by
129
+ return results unless by.any?
130
+ results.map { |x| masher.mash by, x }
131
+ end
132
+
133
+ def merge_these_results results, event, by
134
+ keys_to_merge = convert_this_to_an_array by
135
+ keys_to_merge = event.keys.map { |x| x.to_s } if (keys_to_merge[0] == '*')
136
+
137
+ results.map do |result|
138
+ event.select { |k, _| keys_to_merge.include? k.to_s }.each do |k, v|
139
+ result[k] = v unless result[k]
140
+ end
141
+ result
142
+ end
143
+ end
144
+
145
+ def convert_this_to_an_array value
146
+ [value]
147
+ .flatten
148
+ .map { |x| x.to_s.split(',').map { |x| x.strip } }
149
+ .flatten
150
+ .select { |x| x && x != '' }
151
+ end
152
+
153
+ def convert_to_symbolized_hash event
154
+ data = SymbolizedHash.new
155
+ event.each { |k, v| data[k] = v }
156
+ data
157
+ end
158
+
159
+ def process event, config
160
+ event
161
+ end
162
+
163
+ end
164
+
165
+ end
@@ -0,0 +1,41 @@
1
+ module Mushy
2
+
3
+ class Bash < Flux
4
+
5
+ def self.details
6
+ {
7
+ name: 'Bash',
8
+ description: 'Run a bash command.',
9
+ config: {
10
+ command: {
11
+ description: 'The command to run in bash.',
12
+ type: 'text',
13
+ value: '{{command}}',
14
+ },
15
+ directory: {
16
+ description: 'The working directory in which the command will be run.',
17
+ type: 'text',
18
+ value: '',
19
+ },
20
+ },
21
+ }
22
+ end
23
+
24
+ def process event, config
25
+ command = config[:command]
26
+
27
+ command = "cd #{config[:directory]};#{command}" if config[:directory]
28
+
29
+ text = `#{command}`
30
+
31
+ result = $?
32
+ {
33
+ text: text,
34
+ success: result.success?,
35
+ exit_code: result.to_i,
36
+ }
37
+ end
38
+
39
+ end
40
+
41
+ end
@@ -0,0 +1,106 @@
1
+ require 'ferrum'
2
+
3
+ module Mushy
4
+
5
+ class Browser < Flux
6
+
7
+ def self.details
8
+ {
9
+ name: 'Browser',
10
+ description: 'Visit a page in a browser.',
11
+ config: {
12
+ url: {
13
+ description: 'The URL to visit.',
14
+ type: 'text',
15
+ value: 'https://www.google.com',
16
+ },
17
+ headless: {
18
+ description: 'Run this browser headless.',
19
+ type: 'boolean',
20
+ value: 'true',
21
+ },
22
+ execute: {
23
+ description: 'Javascript to run after the page is loaded.',
24
+ type: 'textarea',
25
+ value: '',
26
+ },
27
+ cookies: {
28
+ description: 'Cookies for the web request. These can be received from a previous browser event with {{cookies}}, or can be typed manually.',
29
+ type: 'editgrid',
30
+ value: [],
31
+ editors: [
32
+ { id: 'name', target: 'name', field: { type: 'text', value: '', default: '' } },
33
+ { id: 'value', target: 'value', field: { type: 'text', value: '', default: '' } },
34
+ { id: 'domain', target: 'domain', field: { type: 'text', value: '', default: '' } },
35
+ { id: 'path', target: 'path', field: { type: 'text', value: '', default: '' } },
36
+ { id: 'expires', target: 'expires', field: { type: 'text', value: '', default: '' } },
37
+ { id: 'size', target: 'size', field: { type: 'integer', value: 0, default: 0 } },
38
+ { id: 'httpOnly', target: 'httpOnly', field: { type: 'boolean', value: false, default: false } },
39
+ { id: 'secure', target: 'secure', field: { type: 'boolean', value: true, default: true } },
40
+ { id: 'sameSite', target: 'sameSite', field: { type: 'text', value: 'None', default: 'None' } },
41
+ { id: 'priority', target: 'priority', field: { type: 'text', value: 'Medium', default: 'Medium' } },
42
+ ],
43
+ #{"name":"1P_JAR","value":"2021-01-21-13","domain":".google.com","path":"/","expires":1613828458.870408,"size":19,"httpOnly":false,"secure":true,"session":false,"sameSite":"None","priority":"Medium"
44
+ },
45
+ carry_cookies_from: {
46
+ description: 'Carry the cookies from this path in the event.',
47
+ type: 'text',
48
+ value: 'cookies',
49
+ },
50
+ headers: {
51
+ description: 'Headers for the web request. These can be received from a previous browser event with {{headers}}, or can be typed manually.',
52
+ type: 'keyvalue',
53
+ value: {},
54
+ },
55
+ carry_headers_from: {
56
+ description: 'Carry the headers from this path in the event.',
57
+ type: 'text',
58
+ value: 'headers',
59
+ },
60
+ },
61
+ }
62
+ end
63
+
64
+ def process event, config
65
+
66
+ browser = Ferrum::Browser.new(headless: (config[:headless].to_s != 'false'))
67
+
68
+ get_the_cookies_from(event, config).each { |c| browser.cookies.set(c) }
69
+
70
+ browser.headers.add get_the_headers_from(event, config)
71
+
72
+ browser.goto config[:url]
73
+
74
+ browser.execute(config[:execute]) if config[:execute]
75
+
76
+ result = {
77
+ url: browser.url,
78
+ cookies: browser.cookies.all.map { |k, v| v.instance_variable_get('@attributes') },
79
+ headers: browser.headers.get,
80
+ body: browser.body
81
+ }
82
+
83
+ browser.quit
84
+
85
+ result
86
+ end
87
+
88
+ def get_the_cookies_from event, config
89
+ cookies = (event[config[:carry_cookies_from].to_sym])
90
+ cookies = [] unless cookies.is_a?(Array)
91
+ config[:cookies] = [] unless config[:cookies].is_a?(Array)
92
+ config[:cookies].each { |x| cookies << x }
93
+ cookies
94
+ end
95
+
96
+ def get_the_headers_from event, config
97
+ headers = (event[config[:carry_headers_from].to_sym])
98
+ headers = {} unless headers.is_a?(Hash)
99
+ config[:headers] = {} unless config[:headers].is_a?(Hash)
100
+ config[:headers].each { |k, v| headers[k] = v }
101
+ headers
102
+ end
103
+
104
+ end
105
+
106
+ end
@@ -0,0 +1,53 @@
1
+ require 'csv'
2
+
3
+ module Mushy
4
+
5
+ class BuildCsv < Flux
6
+
7
+ def self.details
8
+ {
9
+ name: 'BuildCsv',
10
+ description: 'Build a CSV.',
11
+ config: {
12
+ input_path: {
13
+ description: 'The path to the set of records to include in the CSV.',
14
+ type: 'text',
15
+ value: 'records',
16
+ },
17
+ output_path: {
18
+ description: 'The path to the CSV content in the outgoing event.',
19
+ type: 'text',
20
+ value: 'records',
21
+ },
22
+ headers: {
23
+ description: 'The values to include in the CSV, as well as the header values.',
24
+ type: 'keyvalue',
25
+ value: {},
26
+ },
27
+ header_row: {
28
+ description: 'Include a header row?',
29
+ type: 'boolean',
30
+ value: true,
31
+ },
32
+ },
33
+ }
34
+ end
35
+
36
+ def process event, config
37
+ records = event[config[:input_path].to_sym] || event[config[:input_path].to_s]
38
+
39
+ headers = config[:headers]
40
+
41
+ {
42
+ config[:output_path] => CSV.generate do |c|
43
+ if config[:header_row].to_s == 'true'
44
+ c << headers.map { |h| h[1] }
45
+ end
46
+ records.each { |x| c << headers.map { |h| x[h[0].to_sym] || x[h[0].to_s] } }
47
+ end
48
+ }
49
+ end
50
+
51
+ end
52
+
53
+ end