mushy 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/bin/mushy +3 -0
- data/lib/mushy.rb +15 -0
- data/lib/mushy/event.rb +15 -0
- data/lib/mushy/flow.rb +47 -0
- data/lib/mushy/flux.rb +165 -0
- data/lib/mushy/fluxs/bash.rb +41 -0
- data/lib/mushy/fluxs/browser.rb +106 -0
- data/lib/mushy/fluxs/build_csv.rb +53 -0
- data/lib/mushy/fluxs/collection.rb +73 -0
- data/lib/mushy/fluxs/filter.rb +37 -0
- data/lib/mushy/fluxs/get.rb +30 -0
- data/lib/mushy/fluxs/ls.rb +38 -0
- data/lib/mushy/fluxs/parse_html.rb +52 -0
- data/lib/mushy/fluxs/read_csv.rb +27 -0
- data/lib/mushy/fluxs/write_file.rb +39 -0
- data/lib/mushy/masher.rb +56 -0
- data/lib/mushy/run.rb +8 -0
- data/lib/mushy/runner.rb +56 -0
- data/lib/mushy/version.rb +3 -0
- data/lib/public/index.html +256 -0
- data/lib/site.rb +54 -0
- data/mushy.gemspec +24 -0
- data/note.txt +2 -0
- metadata +137 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 4983e312423619f6384adcf954591ebc267020291d05ded5ac5b4f4c3a5e66f5
|
4
|
+
data.tar.gz: ff7265cafa27a7ef94be0e4c99c7e7be0f46448a32efc5b66fa4e6c25739378e
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 6490c4b94d2de183f948ca3cdf70f30e7000ed191b061f427aba15e367c665428cb3aac9af4a95b892c1de01672bef1709e47a48bda41b4391864f2a0af6c35f
|
7
|
+
data.tar.gz: 53b0e05775e441c657711106858fc38a03709ffc36a0bd5430301c2a067aa785d931439f2e2d3d9b46a267d02e841afcbb5128800bbf77ff6dd54bd51a97cbd2
|
data/bin/mushy
ADDED
data/lib/mushy.rb
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'liquid'
|
2
|
+
require 'symbolized'
|
3
|
+
|
4
|
+
Dir[File.dirname(__FILE__) + '/mushy/*.rb'].each { |f| require f }
|
5
|
+
|
6
|
+
important_flux_files = ['bash'].map { |x| "#{x}.rb" }
|
7
|
+
Dir[File.dirname(__FILE__) + '/mushy/fluxs/*.rb']
|
8
|
+
.sort_by { |f| important_flux_files.any? { |x| f.end_with?(x) } ? 0 : 1 }
|
9
|
+
.each { |f| require f }
|
10
|
+
|
11
|
+
module Mushy
|
12
|
+
def self.hi
|
13
|
+
puts 'hello'
|
14
|
+
end
|
15
|
+
end
|
data/lib/mushy/event.rb
ADDED
data/lib/mushy/flow.rb
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
module Mushy
|
4
|
+
|
5
|
+
class Flow
|
6
|
+
attr_accessor :id
|
7
|
+
attr_accessor :fluxs
|
8
|
+
|
9
|
+
def initialize
|
10
|
+
self.id = SecureRandom.uuid
|
11
|
+
self.fluxs = []
|
12
|
+
end
|
13
|
+
|
14
|
+
def fluxs_for event
|
15
|
+
fluxs
|
16
|
+
.select { |x| x.parent_fluxs.any? { |y| y.id == event.flux_id } }
|
17
|
+
.flatten
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.build_flux record
|
21
|
+
flux = Object.const_get("Mushy::#{record[:type] || record['type'] || 'Flux'}").new
|
22
|
+
flux.id = record[:id] || record['id'] || flux.id
|
23
|
+
flux.config = SymbolizedHash.new(record[:config] || record['config'])
|
24
|
+
flux
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.parse data
|
28
|
+
data = JSON.parse data
|
29
|
+
flow = new
|
30
|
+
|
31
|
+
data_fluxs = data['fluxs'] || []
|
32
|
+
|
33
|
+
flow.fluxs = data_fluxs.map { |s| build_flux s }
|
34
|
+
|
35
|
+
fluxs_with_parent_ids = flow.fluxs.reduce({}) { |t, i| t[i.id] = []; t }
|
36
|
+
data_fluxs.map { |r| fluxs_with_parent_ids[r['id']] = r['parent_fluxs'] || [] }
|
37
|
+
|
38
|
+
flow.fluxs.each do |flux|
|
39
|
+
flux.parent_fluxs = flow.fluxs.select { |x| fluxs_with_parent_ids[flux.id].include?(x.id) }
|
40
|
+
end
|
41
|
+
|
42
|
+
flow
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
46
|
+
|
47
|
+
end
|
data/lib/mushy/flux.rb
ADDED
@@ -0,0 +1,165 @@
|
|
1
|
+
module Mushy
|
2
|
+
|
3
|
+
class Flux
|
4
|
+
|
5
|
+
attr_accessor :id
|
6
|
+
attr_accessor :parent_fluxs
|
7
|
+
attr_accessor :subscribed_to
|
8
|
+
attr_accessor :config
|
9
|
+
attr_accessor :masher
|
10
|
+
|
11
|
+
def initialize
|
12
|
+
guard
|
13
|
+
end
|
14
|
+
|
15
|
+
class << self
|
16
|
+
attr_accessor :all
|
17
|
+
|
18
|
+
def inherited subclass
|
19
|
+
if (self != Mushy::Flux)
|
20
|
+
Mushy::Flux.inherited subclass
|
21
|
+
else
|
22
|
+
self.all ||= []
|
23
|
+
self.all << subclass
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def guard
|
29
|
+
self.id ||= SecureRandom.uuid
|
30
|
+
self.parent_fluxs ||= []
|
31
|
+
self.subscribed_to ||= []
|
32
|
+
self.masher ||= Masher.new
|
33
|
+
self.config ||= SymbolizedHash.new
|
34
|
+
end
|
35
|
+
|
36
|
+
def execute incoming_event
|
37
|
+
guard
|
38
|
+
|
39
|
+
incoming_event = SymbolizedHash.new(incoming_event) if incoming_event.is_a?(Hash)
|
40
|
+
|
41
|
+
event = incoming_event
|
42
|
+
|
43
|
+
mashed_config = masher.mash config, event
|
44
|
+
|
45
|
+
incoming_split = mashed_config[:incoming_split]
|
46
|
+
|
47
|
+
events = incoming_split ? incoming_event[incoming_split] : [event]
|
48
|
+
|
49
|
+
results = events.map { |e| execute_single_event e, config }
|
50
|
+
|
51
|
+
return results.first unless incoming_split
|
52
|
+
|
53
|
+
results = join_these_results(results, event, config[:join]) if config[:join]
|
54
|
+
|
55
|
+
results.flatten
|
56
|
+
end
|
57
|
+
|
58
|
+
def execute_single_event event, config
|
59
|
+
|
60
|
+
mashed_config = masher.mash config, event
|
61
|
+
|
62
|
+
the_original_join = mashed_config[:join]
|
63
|
+
mashed_config[:join] = nil if mashed_config[:incoming_split]
|
64
|
+
|
65
|
+
results = process event, mashed_config
|
66
|
+
|
67
|
+
returned_one_result = results.is_a?(Hash)
|
68
|
+
|
69
|
+
results = standardize_these results
|
70
|
+
results = shape_these results, event, mashed_config
|
71
|
+
|
72
|
+
return results.first if the_original_join
|
73
|
+
|
74
|
+
return results if mashed_config[:outgoing_split]
|
75
|
+
|
76
|
+
returned_one_result ? results.first : results
|
77
|
+
|
78
|
+
end
|
79
|
+
|
80
|
+
def standardize_these results
|
81
|
+
[results]
|
82
|
+
.flatten
|
83
|
+
.map { |x| x.is_a?(Hash) ? convert_to_symbolized_hash(x) : nil }
|
84
|
+
.select { |x| x }
|
85
|
+
end
|
86
|
+
|
87
|
+
def shape_these results, event, config
|
88
|
+
supported_shaping = [:merge, :outgoing_split, :group, :model, :join, :sort, :limit]
|
89
|
+
|
90
|
+
shaping = supported_shaping
|
91
|
+
if (config[:shaping])
|
92
|
+
shaping = convert_this_to_an_array(config[:shaping]).map { |x| x.to_sym }
|
93
|
+
end
|
94
|
+
|
95
|
+
supported_shaping
|
96
|
+
.select { |x| config[x] }
|
97
|
+
.each_with_index
|
98
|
+
.sort_by { |x, i| shaping.index(x) || i + supported_shaping.count }
|
99
|
+
.map { |x, _| x }
|
100
|
+
.reduce(results) { |t, i| self.send("#{i}_these_results".to_sym, t, event, config[i]) }
|
101
|
+
end
|
102
|
+
|
103
|
+
def sort_these_results results, event, by
|
104
|
+
results.sort { |x| x[by].to_i }
|
105
|
+
end
|
106
|
+
|
107
|
+
def limit_these_results results, event, by
|
108
|
+
results
|
109
|
+
.each_with_index
|
110
|
+
.select { |x, i| i < by.to_i }
|
111
|
+
.map { |x, _| x }
|
112
|
+
end
|
113
|
+
|
114
|
+
def group_these_results results, event, by
|
115
|
+
group_by = by.split('|')[0]
|
116
|
+
result_key = by.split('|')[1]
|
117
|
+
results.group_by { |x| x[group_by] }.map { |k, v| SymbolizedHash.new( { result_key => v } ) }
|
118
|
+
end
|
119
|
+
|
120
|
+
def outgoing_split_these_results results, event, by
|
121
|
+
results.map { |x| Masher.new.dig by, x }.flatten
|
122
|
+
end
|
123
|
+
|
124
|
+
def join_these_results results, event, by
|
125
|
+
[SymbolizedHash.new( { by => results } )]
|
126
|
+
end
|
127
|
+
|
128
|
+
def model_these_results results, event, by
|
129
|
+
return results unless by.any?
|
130
|
+
results.map { |x| masher.mash by, x }
|
131
|
+
end
|
132
|
+
|
133
|
+
def merge_these_results results, event, by
|
134
|
+
keys_to_merge = convert_this_to_an_array by
|
135
|
+
keys_to_merge = event.keys.map { |x| x.to_s } if (keys_to_merge[0] == '*')
|
136
|
+
|
137
|
+
results.map do |result|
|
138
|
+
event.select { |k, _| keys_to_merge.include? k.to_s }.each do |k, v|
|
139
|
+
result[k] = v unless result[k]
|
140
|
+
end
|
141
|
+
result
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
def convert_this_to_an_array value
|
146
|
+
[value]
|
147
|
+
.flatten
|
148
|
+
.map { |x| x.to_s.split(',').map { |x| x.strip } }
|
149
|
+
.flatten
|
150
|
+
.select { |x| x && x != '' }
|
151
|
+
end
|
152
|
+
|
153
|
+
def convert_to_symbolized_hash event
|
154
|
+
data = SymbolizedHash.new
|
155
|
+
event.each { |k, v| data[k] = v }
|
156
|
+
data
|
157
|
+
end
|
158
|
+
|
159
|
+
def process event, config
|
160
|
+
event
|
161
|
+
end
|
162
|
+
|
163
|
+
end
|
164
|
+
|
165
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
module Mushy
|
2
|
+
|
3
|
+
class Bash < Flux
|
4
|
+
|
5
|
+
def self.details
|
6
|
+
{
|
7
|
+
name: 'Bash',
|
8
|
+
description: 'Run a bash command.',
|
9
|
+
config: {
|
10
|
+
command: {
|
11
|
+
description: 'The command to run in bash.',
|
12
|
+
type: 'text',
|
13
|
+
value: '{{command}}',
|
14
|
+
},
|
15
|
+
directory: {
|
16
|
+
description: 'The working directory in which the command will be run.',
|
17
|
+
type: 'text',
|
18
|
+
value: '',
|
19
|
+
},
|
20
|
+
},
|
21
|
+
}
|
22
|
+
end
|
23
|
+
|
24
|
+
def process event, config
|
25
|
+
command = config[:command]
|
26
|
+
|
27
|
+
command = "cd #{config[:directory]};#{command}" if config[:directory]
|
28
|
+
|
29
|
+
text = `#{command}`
|
30
|
+
|
31
|
+
result = $?
|
32
|
+
{
|
33
|
+
text: text,
|
34
|
+
success: result.success?,
|
35
|
+
exit_code: result.to_i,
|
36
|
+
}
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
@@ -0,0 +1,106 @@
|
|
1
|
+
require 'ferrum'
|
2
|
+
|
3
|
+
module Mushy
|
4
|
+
|
5
|
+
class Browser < Flux
|
6
|
+
|
7
|
+
def self.details
|
8
|
+
{
|
9
|
+
name: 'Browser',
|
10
|
+
description: 'Visit a page in a browser.',
|
11
|
+
config: {
|
12
|
+
url: {
|
13
|
+
description: 'The URL to visit.',
|
14
|
+
type: 'text',
|
15
|
+
value: 'https://www.google.com',
|
16
|
+
},
|
17
|
+
headless: {
|
18
|
+
description: 'Run this browser headless.',
|
19
|
+
type: 'boolean',
|
20
|
+
value: 'true',
|
21
|
+
},
|
22
|
+
execute: {
|
23
|
+
description: 'Javascript to run after the page is loaded.',
|
24
|
+
type: 'textarea',
|
25
|
+
value: '',
|
26
|
+
},
|
27
|
+
cookies: {
|
28
|
+
description: 'Cookies for the web request. These can be received from a previous browser event with {{cookies}}, or can be typed manually.',
|
29
|
+
type: 'editgrid',
|
30
|
+
value: [],
|
31
|
+
editors: [
|
32
|
+
{ id: 'name', target: 'name', field: { type: 'text', value: '', default: '' } },
|
33
|
+
{ id: 'value', target: 'value', field: { type: 'text', value: '', default: '' } },
|
34
|
+
{ id: 'domain', target: 'domain', field: { type: 'text', value: '', default: '' } },
|
35
|
+
{ id: 'path', target: 'path', field: { type: 'text', value: '', default: '' } },
|
36
|
+
{ id: 'expires', target: 'expires', field: { type: 'text', value: '', default: '' } },
|
37
|
+
{ id: 'size', target: 'size', field: { type: 'integer', value: 0, default: 0 } },
|
38
|
+
{ id: 'httpOnly', target: 'httpOnly', field: { type: 'boolean', value: false, default: false } },
|
39
|
+
{ id: 'secure', target: 'secure', field: { type: 'boolean', value: true, default: true } },
|
40
|
+
{ id: 'sameSite', target: 'sameSite', field: { type: 'text', value: 'None', default: 'None' } },
|
41
|
+
{ id: 'priority', target: 'priority', field: { type: 'text', value: 'Medium', default: 'Medium' } },
|
42
|
+
],
|
43
|
+
#{"name":"1P_JAR","value":"2021-01-21-13","domain":".google.com","path":"/","expires":1613828458.870408,"size":19,"httpOnly":false,"secure":true,"session":false,"sameSite":"None","priority":"Medium"
|
44
|
+
},
|
45
|
+
carry_cookies_from: {
|
46
|
+
description: 'Carry the cookies from this path in the event.',
|
47
|
+
type: 'text',
|
48
|
+
value: 'cookies',
|
49
|
+
},
|
50
|
+
headers: {
|
51
|
+
description: 'Headers for the web request. These can be received from a previous browser event with {{headers}}, or can be typed manually.',
|
52
|
+
type: 'keyvalue',
|
53
|
+
value: {},
|
54
|
+
},
|
55
|
+
carry_headers_from: {
|
56
|
+
description: 'Carry the headers from this path in the event.',
|
57
|
+
type: 'text',
|
58
|
+
value: 'headers',
|
59
|
+
},
|
60
|
+
},
|
61
|
+
}
|
62
|
+
end
|
63
|
+
|
64
|
+
def process event, config
|
65
|
+
|
66
|
+
browser = Ferrum::Browser.new(headless: (config[:headless].to_s != 'false'))
|
67
|
+
|
68
|
+
get_the_cookies_from(event, config).each { |c| browser.cookies.set(c) }
|
69
|
+
|
70
|
+
browser.headers.add get_the_headers_from(event, config)
|
71
|
+
|
72
|
+
browser.goto config[:url]
|
73
|
+
|
74
|
+
browser.execute(config[:execute]) if config[:execute]
|
75
|
+
|
76
|
+
result = {
|
77
|
+
url: browser.url,
|
78
|
+
cookies: browser.cookies.all.map { |k, v| v.instance_variable_get('@attributes') },
|
79
|
+
headers: browser.headers.get,
|
80
|
+
body: browser.body
|
81
|
+
}
|
82
|
+
|
83
|
+
browser.quit
|
84
|
+
|
85
|
+
result
|
86
|
+
end
|
87
|
+
|
88
|
+
def get_the_cookies_from event, config
|
89
|
+
cookies = (event[config[:carry_cookies_from].to_sym])
|
90
|
+
cookies = [] unless cookies.is_a?(Array)
|
91
|
+
config[:cookies] = [] unless config[:cookies].is_a?(Array)
|
92
|
+
config[:cookies].each { |x| cookies << x }
|
93
|
+
cookies
|
94
|
+
end
|
95
|
+
|
96
|
+
def get_the_headers_from event, config
|
97
|
+
headers = (event[config[:carry_headers_from].to_sym])
|
98
|
+
headers = {} unless headers.is_a?(Hash)
|
99
|
+
config[:headers] = {} unless config[:headers].is_a?(Hash)
|
100
|
+
config[:headers].each { |k, v| headers[k] = v }
|
101
|
+
headers
|
102
|
+
end
|
103
|
+
|
104
|
+
end
|
105
|
+
|
106
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
require 'csv'
|
2
|
+
|
3
|
+
module Mushy
|
4
|
+
|
5
|
+
class BuildCsv < Flux
|
6
|
+
|
7
|
+
def self.details
|
8
|
+
{
|
9
|
+
name: 'BuildCsv',
|
10
|
+
description: 'Build a CSV.',
|
11
|
+
config: {
|
12
|
+
input_path: {
|
13
|
+
description: 'The path to the set of records to include in the CSV.',
|
14
|
+
type: 'text',
|
15
|
+
value: 'records',
|
16
|
+
},
|
17
|
+
output_path: {
|
18
|
+
description: 'The path to the CSV content in the outgoing event.',
|
19
|
+
type: 'text',
|
20
|
+
value: 'records',
|
21
|
+
},
|
22
|
+
headers: {
|
23
|
+
description: 'The values to include in the CSV, as well as the header values.',
|
24
|
+
type: 'keyvalue',
|
25
|
+
value: {},
|
26
|
+
},
|
27
|
+
header_row: {
|
28
|
+
description: 'Include a header row?',
|
29
|
+
type: 'boolean',
|
30
|
+
value: true,
|
31
|
+
},
|
32
|
+
},
|
33
|
+
}
|
34
|
+
end
|
35
|
+
|
36
|
+
def process event, config
|
37
|
+
records = event[config[:input_path].to_sym] || event[config[:input_path].to_s]
|
38
|
+
|
39
|
+
headers = config[:headers]
|
40
|
+
|
41
|
+
{
|
42
|
+
config[:output_path] => CSV.generate do |c|
|
43
|
+
if config[:header_row].to_s == 'true'
|
44
|
+
c << headers.map { |h| h[1] }
|
45
|
+
end
|
46
|
+
records.each { |x| c << headers.map { |h| x[h[0].to_sym] || x[h[0].to_s] } }
|
47
|
+
end
|
48
|
+
}
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|