mushy 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/bin/mushy +3 -0
- data/lib/mushy.rb +15 -0
- data/lib/mushy/event.rb +15 -0
- data/lib/mushy/flow.rb +47 -0
- data/lib/mushy/flux.rb +165 -0
- data/lib/mushy/fluxs/bash.rb +41 -0
- data/lib/mushy/fluxs/browser.rb +106 -0
- data/lib/mushy/fluxs/build_csv.rb +53 -0
- data/lib/mushy/fluxs/collection.rb +73 -0
- data/lib/mushy/fluxs/filter.rb +37 -0
- data/lib/mushy/fluxs/get.rb +30 -0
- data/lib/mushy/fluxs/ls.rb +38 -0
- data/lib/mushy/fluxs/parse_html.rb +52 -0
- data/lib/mushy/fluxs/read_csv.rb +27 -0
- data/lib/mushy/fluxs/write_file.rb +39 -0
- data/lib/mushy/masher.rb +56 -0
- data/lib/mushy/run.rb +8 -0
- data/lib/mushy/runner.rb +56 -0
- data/lib/mushy/version.rb +3 -0
- data/lib/public/index.html +256 -0
- data/lib/site.rb +54 -0
- data/mushy.gemspec +24 -0
- data/note.txt +2 -0
- metadata +137 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 4983e312423619f6384adcf954591ebc267020291d05ded5ac5b4f4c3a5e66f5
|
4
|
+
data.tar.gz: ff7265cafa27a7ef94be0e4c99c7e7be0f46448a32efc5b66fa4e6c25739378e
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 6490c4b94d2de183f948ca3cdf70f30e7000ed191b061f427aba15e367c665428cb3aac9af4a95b892c1de01672bef1709e47a48bda41b4391864f2a0af6c35f
|
7
|
+
data.tar.gz: 53b0e05775e441c657711106858fc38a03709ffc36a0bd5430301c2a067aa785d931439f2e2d3d9b46a267d02e841afcbb5128800bbf77ff6dd54bd51a97cbd2
|
data/bin/mushy
ADDED
data/lib/mushy.rb
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'liquid'
|
2
|
+
require 'symbolized'
|
3
|
+
|
4
|
+
Dir[File.dirname(__FILE__) + '/mushy/*.rb'].each { |f| require f }
|
5
|
+
|
6
|
+
important_flux_files = ['bash'].map { |x| "#{x}.rb" }
|
7
|
+
Dir[File.dirname(__FILE__) + '/mushy/fluxs/*.rb']
|
8
|
+
.sort_by { |f| important_flux_files.any? { |x| f.end_with?(x) } ? 0 : 1 }
|
9
|
+
.each { |f| require f }
|
10
|
+
|
11
|
+
module Mushy
|
12
|
+
def self.hi
|
13
|
+
puts 'hello'
|
14
|
+
end
|
15
|
+
end
|
data/lib/mushy/event.rb
ADDED
data/lib/mushy/flow.rb
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
module Mushy
|
4
|
+
|
5
|
+
class Flow
|
6
|
+
attr_accessor :id
|
7
|
+
attr_accessor :fluxs
|
8
|
+
|
9
|
+
def initialize
|
10
|
+
self.id = SecureRandom.uuid
|
11
|
+
self.fluxs = []
|
12
|
+
end
|
13
|
+
|
14
|
+
def fluxs_for event
|
15
|
+
fluxs
|
16
|
+
.select { |x| x.parent_fluxs.any? { |y| y.id == event.flux_id } }
|
17
|
+
.flatten
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.build_flux record
|
21
|
+
flux = Object.const_get("Mushy::#{record[:type] || record['type'] || 'Flux'}").new
|
22
|
+
flux.id = record[:id] || record['id'] || flux.id
|
23
|
+
flux.config = SymbolizedHash.new(record[:config] || record['config'])
|
24
|
+
flux
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.parse data
|
28
|
+
data = JSON.parse data
|
29
|
+
flow = new
|
30
|
+
|
31
|
+
data_fluxs = data['fluxs'] || []
|
32
|
+
|
33
|
+
flow.fluxs = data_fluxs.map { |s| build_flux s }
|
34
|
+
|
35
|
+
fluxs_with_parent_ids = flow.fluxs.reduce({}) { |t, i| t[i.id] = []; t }
|
36
|
+
data_fluxs.map { |r| fluxs_with_parent_ids[r['id']] = r['parent_fluxs'] || [] }
|
37
|
+
|
38
|
+
flow.fluxs.each do |flux|
|
39
|
+
flux.parent_fluxs = flow.fluxs.select { |x| fluxs_with_parent_ids[flux.id].include?(x.id) }
|
40
|
+
end
|
41
|
+
|
42
|
+
flow
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
46
|
+
|
47
|
+
end
|
data/lib/mushy/flux.rb
ADDED
@@ -0,0 +1,165 @@
|
|
1
|
+
module Mushy
|
2
|
+
|
3
|
+
class Flux
|
4
|
+
|
5
|
+
attr_accessor :id
|
6
|
+
attr_accessor :parent_fluxs
|
7
|
+
attr_accessor :subscribed_to
|
8
|
+
attr_accessor :config
|
9
|
+
attr_accessor :masher
|
10
|
+
|
11
|
+
def initialize
|
12
|
+
guard
|
13
|
+
end
|
14
|
+
|
15
|
+
class << self
|
16
|
+
attr_accessor :all
|
17
|
+
|
18
|
+
def inherited subclass
|
19
|
+
if (self != Mushy::Flux)
|
20
|
+
Mushy::Flux.inherited subclass
|
21
|
+
else
|
22
|
+
self.all ||= []
|
23
|
+
self.all << subclass
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def guard
|
29
|
+
self.id ||= SecureRandom.uuid
|
30
|
+
self.parent_fluxs ||= []
|
31
|
+
self.subscribed_to ||= []
|
32
|
+
self.masher ||= Masher.new
|
33
|
+
self.config ||= SymbolizedHash.new
|
34
|
+
end
|
35
|
+
|
36
|
+
def execute incoming_event
|
37
|
+
guard
|
38
|
+
|
39
|
+
incoming_event = SymbolizedHash.new(incoming_event) if incoming_event.is_a?(Hash)
|
40
|
+
|
41
|
+
event = incoming_event
|
42
|
+
|
43
|
+
mashed_config = masher.mash config, event
|
44
|
+
|
45
|
+
incoming_split = mashed_config[:incoming_split]
|
46
|
+
|
47
|
+
events = incoming_split ? incoming_event[incoming_split] : [event]
|
48
|
+
|
49
|
+
results = events.map { |e| execute_single_event e, config }
|
50
|
+
|
51
|
+
return results.first unless incoming_split
|
52
|
+
|
53
|
+
results = join_these_results(results, event, config[:join]) if config[:join]
|
54
|
+
|
55
|
+
results.flatten
|
56
|
+
end
|
57
|
+
|
58
|
+
def execute_single_event event, config
|
59
|
+
|
60
|
+
mashed_config = masher.mash config, event
|
61
|
+
|
62
|
+
the_original_join = mashed_config[:join]
|
63
|
+
mashed_config[:join] = nil if mashed_config[:incoming_split]
|
64
|
+
|
65
|
+
results = process event, mashed_config
|
66
|
+
|
67
|
+
returned_one_result = results.is_a?(Hash)
|
68
|
+
|
69
|
+
results = standardize_these results
|
70
|
+
results = shape_these results, event, mashed_config
|
71
|
+
|
72
|
+
return results.first if the_original_join
|
73
|
+
|
74
|
+
return results if mashed_config[:outgoing_split]
|
75
|
+
|
76
|
+
returned_one_result ? results.first : results
|
77
|
+
|
78
|
+
end
|
79
|
+
|
80
|
+
def standardize_these results
|
81
|
+
[results]
|
82
|
+
.flatten
|
83
|
+
.map { |x| x.is_a?(Hash) ? convert_to_symbolized_hash(x) : nil }
|
84
|
+
.select { |x| x }
|
85
|
+
end
|
86
|
+
|
87
|
+
def shape_these results, event, config
|
88
|
+
supported_shaping = [:merge, :outgoing_split, :group, :model, :join, :sort, :limit]
|
89
|
+
|
90
|
+
shaping = supported_shaping
|
91
|
+
if (config[:shaping])
|
92
|
+
shaping = convert_this_to_an_array(config[:shaping]).map { |x| x.to_sym }
|
93
|
+
end
|
94
|
+
|
95
|
+
supported_shaping
|
96
|
+
.select { |x| config[x] }
|
97
|
+
.each_with_index
|
98
|
+
.sort_by { |x, i| shaping.index(x) || i + supported_shaping.count }
|
99
|
+
.map { |x, _| x }
|
100
|
+
.reduce(results) { |t, i| self.send("#{i}_these_results".to_sym, t, event, config[i]) }
|
101
|
+
end
|
102
|
+
|
103
|
+
def sort_these_results results, event, by
|
104
|
+
results.sort { |x| x[by].to_i }
|
105
|
+
end
|
106
|
+
|
107
|
+
def limit_these_results results, event, by
|
108
|
+
results
|
109
|
+
.each_with_index
|
110
|
+
.select { |x, i| i < by.to_i }
|
111
|
+
.map { |x, _| x }
|
112
|
+
end
|
113
|
+
|
114
|
+
def group_these_results results, event, by
|
115
|
+
group_by = by.split('|')[0]
|
116
|
+
result_key = by.split('|')[1]
|
117
|
+
results.group_by { |x| x[group_by] }.map { |k, v| SymbolizedHash.new( { result_key => v } ) }
|
118
|
+
end
|
119
|
+
|
120
|
+
def outgoing_split_these_results results, event, by
|
121
|
+
results.map { |x| Masher.new.dig by, x }.flatten
|
122
|
+
end
|
123
|
+
|
124
|
+
def join_these_results results, event, by
|
125
|
+
[SymbolizedHash.new( { by => results } )]
|
126
|
+
end
|
127
|
+
|
128
|
+
def model_these_results results, event, by
|
129
|
+
return results unless by.any?
|
130
|
+
results.map { |x| masher.mash by, x }
|
131
|
+
end
|
132
|
+
|
133
|
+
def merge_these_results results, event, by
|
134
|
+
keys_to_merge = convert_this_to_an_array by
|
135
|
+
keys_to_merge = event.keys.map { |x| x.to_s } if (keys_to_merge[0] == '*')
|
136
|
+
|
137
|
+
results.map do |result|
|
138
|
+
event.select { |k, _| keys_to_merge.include? k.to_s }.each do |k, v|
|
139
|
+
result[k] = v unless result[k]
|
140
|
+
end
|
141
|
+
result
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
def convert_this_to_an_array value
|
146
|
+
[value]
|
147
|
+
.flatten
|
148
|
+
.map { |x| x.to_s.split(',').map { |x| x.strip } }
|
149
|
+
.flatten
|
150
|
+
.select { |x| x && x != '' }
|
151
|
+
end
|
152
|
+
|
153
|
+
def convert_to_symbolized_hash event
|
154
|
+
data = SymbolizedHash.new
|
155
|
+
event.each { |k, v| data[k] = v }
|
156
|
+
data
|
157
|
+
end
|
158
|
+
|
159
|
+
def process event, config
|
160
|
+
event
|
161
|
+
end
|
162
|
+
|
163
|
+
end
|
164
|
+
|
165
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
module Mushy
|
2
|
+
|
3
|
+
class Bash < Flux
|
4
|
+
|
5
|
+
def self.details
|
6
|
+
{
|
7
|
+
name: 'Bash',
|
8
|
+
description: 'Run a bash command.',
|
9
|
+
config: {
|
10
|
+
command: {
|
11
|
+
description: 'The command to run in bash.',
|
12
|
+
type: 'text',
|
13
|
+
value: '{{command}}',
|
14
|
+
},
|
15
|
+
directory: {
|
16
|
+
description: 'The working directory in which the command will be run.',
|
17
|
+
type: 'text',
|
18
|
+
value: '',
|
19
|
+
},
|
20
|
+
},
|
21
|
+
}
|
22
|
+
end
|
23
|
+
|
24
|
+
def process event, config
|
25
|
+
command = config[:command]
|
26
|
+
|
27
|
+
command = "cd #{config[:directory]};#{command}" if config[:directory]
|
28
|
+
|
29
|
+
text = `#{command}`
|
30
|
+
|
31
|
+
result = $?
|
32
|
+
{
|
33
|
+
text: text,
|
34
|
+
success: result.success?,
|
35
|
+
exit_code: result.to_i,
|
36
|
+
}
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
@@ -0,0 +1,106 @@
|
|
1
|
+
require 'ferrum'
|
2
|
+
|
3
|
+
module Mushy
|
4
|
+
|
5
|
+
class Browser < Flux
|
6
|
+
|
7
|
+
def self.details
|
8
|
+
{
|
9
|
+
name: 'Browser',
|
10
|
+
description: 'Visit a page in a browser.',
|
11
|
+
config: {
|
12
|
+
url: {
|
13
|
+
description: 'The URL to visit.',
|
14
|
+
type: 'text',
|
15
|
+
value: 'https://www.google.com',
|
16
|
+
},
|
17
|
+
headless: {
|
18
|
+
description: 'Run this browser headless.',
|
19
|
+
type: 'boolean',
|
20
|
+
value: 'true',
|
21
|
+
},
|
22
|
+
execute: {
|
23
|
+
description: 'Javascript to run after the page is loaded.',
|
24
|
+
type: 'textarea',
|
25
|
+
value: '',
|
26
|
+
},
|
27
|
+
cookies: {
|
28
|
+
description: 'Cookies for the web request. These can be received from a previous browser event with {{cookies}}, or can be typed manually.',
|
29
|
+
type: 'editgrid',
|
30
|
+
value: [],
|
31
|
+
editors: [
|
32
|
+
{ id: 'name', target: 'name', field: { type: 'text', value: '', default: '' } },
|
33
|
+
{ id: 'value', target: 'value', field: { type: 'text', value: '', default: '' } },
|
34
|
+
{ id: 'domain', target: 'domain', field: { type: 'text', value: '', default: '' } },
|
35
|
+
{ id: 'path', target: 'path', field: { type: 'text', value: '', default: '' } },
|
36
|
+
{ id: 'expires', target: 'expires', field: { type: 'text', value: '', default: '' } },
|
37
|
+
{ id: 'size', target: 'size', field: { type: 'integer', value: 0, default: 0 } },
|
38
|
+
{ id: 'httpOnly', target: 'httpOnly', field: { type: 'boolean', value: false, default: false } },
|
39
|
+
{ id: 'secure', target: 'secure', field: { type: 'boolean', value: true, default: true } },
|
40
|
+
{ id: 'sameSite', target: 'sameSite', field: { type: 'text', value: 'None', default: 'None' } },
|
41
|
+
{ id: 'priority', target: 'priority', field: { type: 'text', value: 'Medium', default: 'Medium' } },
|
42
|
+
],
|
43
|
+
#{"name":"1P_JAR","value":"2021-01-21-13","domain":".google.com","path":"/","expires":1613828458.870408,"size":19,"httpOnly":false,"secure":true,"session":false,"sameSite":"None","priority":"Medium"
|
44
|
+
},
|
45
|
+
carry_cookies_from: {
|
46
|
+
description: 'Carry the cookies from this path in the event.',
|
47
|
+
type: 'text',
|
48
|
+
value: 'cookies',
|
49
|
+
},
|
50
|
+
headers: {
|
51
|
+
description: 'Headers for the web request. These can be received from a previous browser event with {{headers}}, or can be typed manually.',
|
52
|
+
type: 'keyvalue',
|
53
|
+
value: {},
|
54
|
+
},
|
55
|
+
carry_headers_from: {
|
56
|
+
description: 'Carry the headers from this path in the event.',
|
57
|
+
type: 'text',
|
58
|
+
value: 'headers',
|
59
|
+
},
|
60
|
+
},
|
61
|
+
}
|
62
|
+
end
|
63
|
+
|
64
|
+
def process event, config
|
65
|
+
|
66
|
+
browser = Ferrum::Browser.new(headless: (config[:headless].to_s != 'false'))
|
67
|
+
|
68
|
+
get_the_cookies_from(event, config).each { |c| browser.cookies.set(c) }
|
69
|
+
|
70
|
+
browser.headers.add get_the_headers_from(event, config)
|
71
|
+
|
72
|
+
browser.goto config[:url]
|
73
|
+
|
74
|
+
browser.execute(config[:execute]) if config[:execute]
|
75
|
+
|
76
|
+
result = {
|
77
|
+
url: browser.url,
|
78
|
+
cookies: browser.cookies.all.map { |k, v| v.instance_variable_get('@attributes') },
|
79
|
+
headers: browser.headers.get,
|
80
|
+
body: browser.body
|
81
|
+
}
|
82
|
+
|
83
|
+
browser.quit
|
84
|
+
|
85
|
+
result
|
86
|
+
end
|
87
|
+
|
88
|
+
def get_the_cookies_from event, config
|
89
|
+
cookies = (event[config[:carry_cookies_from].to_sym])
|
90
|
+
cookies = [] unless cookies.is_a?(Array)
|
91
|
+
config[:cookies] = [] unless config[:cookies].is_a?(Array)
|
92
|
+
config[:cookies].each { |x| cookies << x }
|
93
|
+
cookies
|
94
|
+
end
|
95
|
+
|
96
|
+
def get_the_headers_from event, config
|
97
|
+
headers = (event[config[:carry_headers_from].to_sym])
|
98
|
+
headers = {} unless headers.is_a?(Hash)
|
99
|
+
config[:headers] = {} unless config[:headers].is_a?(Hash)
|
100
|
+
config[:headers].each { |k, v| headers[k] = v }
|
101
|
+
headers
|
102
|
+
end
|
103
|
+
|
104
|
+
end
|
105
|
+
|
106
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
require 'csv'
|
2
|
+
|
3
|
+
module Mushy
|
4
|
+
|
5
|
+
class BuildCsv < Flux
|
6
|
+
|
7
|
+
def self.details
|
8
|
+
{
|
9
|
+
name: 'BuildCsv',
|
10
|
+
description: 'Build a CSV.',
|
11
|
+
config: {
|
12
|
+
input_path: {
|
13
|
+
description: 'The path to the set of records to include in the CSV.',
|
14
|
+
type: 'text',
|
15
|
+
value: 'records',
|
16
|
+
},
|
17
|
+
output_path: {
|
18
|
+
description: 'The path to the CSV content in the outgoing event.',
|
19
|
+
type: 'text',
|
20
|
+
value: 'records',
|
21
|
+
},
|
22
|
+
headers: {
|
23
|
+
description: 'The values to include in the CSV, as well as the header values.',
|
24
|
+
type: 'keyvalue',
|
25
|
+
value: {},
|
26
|
+
},
|
27
|
+
header_row: {
|
28
|
+
description: 'Include a header row?',
|
29
|
+
type: 'boolean',
|
30
|
+
value: true,
|
31
|
+
},
|
32
|
+
},
|
33
|
+
}
|
34
|
+
end
|
35
|
+
|
36
|
+
def process event, config
|
37
|
+
records = event[config[:input_path].to_sym] || event[config[:input_path].to_s]
|
38
|
+
|
39
|
+
headers = config[:headers]
|
40
|
+
|
41
|
+
{
|
42
|
+
config[:output_path] => CSV.generate do |c|
|
43
|
+
if config[:header_row].to_s == 'true'
|
44
|
+
c << headers.map { |h| h[1] }
|
45
|
+
end
|
46
|
+
records.each { |x| c << headers.map { |h| x[h[0].to_sym] || x[h[0].to_s] } }
|
47
|
+
end
|
48
|
+
}
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|