mushy 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 4983e312423619f6384adcf954591ebc267020291d05ded5ac5b4f4c3a5e66f5
+  data.tar.gz: ff7265cafa27a7ef94be0e4c99c7e7be0f46448a32efc5b66fa4e6c25739378e
+SHA512:
+  metadata.gz: 6490c4b94d2de183f948ca3cdf70f30e7000ed191b061f427aba15e367c665428cb3aac9af4a95b892c1de01672bef1709e47a48bda41b4391864f2a0af6c35f
+  data.tar.gz: 53b0e05775e441c657711106858fc38a03709ffc36a0bd5430301c2a067aa785d931439f2e2d3d9b46a267d02e841afcbb5128800bbf77ff6dd54bd51a97cbd2

data/bin/mushy

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+
+puts 'mushy'

data/lib/mushy.rb

@@ -0,0 +1,15 @@
+require 'liquid'
+require 'symbolized'
+
+Dir[File.dirname(__FILE__) + '/mushy/*.rb'].each { |f| require f }
+
+important_flux_files = ['bash'].map { |x| "#{x}.rb" }
+Dir[File.dirname(__FILE__) + '/mushy/fluxs/*.rb']
+  .sort_by { |f| important_flux_files.any? { |x| f.end_with?(x) } ? 0 : 1 }
+  .each { |f| require f }
+
+module Mushy
+  def self.hi
+    puts 'hello'
+  end
+end

data/lib/mushy/event.rb

@@ -0,0 +1,15 @@
+module Mushy
+
+  class Event
+    attr_accessor :id
+    attr_accessor :run_id
+    attr_accessor :flux_id
+    attr_accessor :flow_id
+    attr_accessor :data
+
+    def initialize
+      self.data = SymbolizedHash.new
+    end
+  end
+
+end

data/lib/mushy/flow.rb

@@ -0,0 +1,47 @@
+require 'json'
+
+module Mushy
+
+  class Flow
+    attr_accessor :id
+    attr_accessor :fluxs
+
+    def initialize
+      self.id = SecureRandom.uuid
+      self.fluxs = []
+    end
+
+    def fluxs_for event
+      fluxs
+        .select { |x| x.parent_fluxs.any? { |y| y.id == event.flux_id } }
+        .flatten
+    end
+
+    def self.build_flux record
+      flux = Object.const_get("Mushy::#{record[:type] || record['type'] || 'Flux'}").new
+      flux.id = record[:id] || record['id'] || flux.id
+      flux.config = SymbolizedHash.new(record[:config] || record['config'])
+      flux
+    end
+
+    def self.parse data
+      data = JSON.parse data
+      flow = new
+
+      data_fluxs = data['fluxs'] || []
+
+      flow.fluxs = data_fluxs.map { |s| build_flux s }
+
+      fluxs_with_parent_ids = flow.fluxs.reduce({}) { |t, i| t[i.id] = []; t }
+      data_fluxs.map { |r| fluxs_with_parent_ids[r['id']] = r['parent_fluxs'] || [] }
+
+      flow.fluxs.each do |flux|
+        flux.parent_fluxs = flow.fluxs.select { |x| fluxs_with_parent_ids[flux.id].include?(x.id) }
+      end
+
+      flow
+    end
+
+  end
+
+end

data/lib/mushy/flux.rb

@@ -0,0 +1,165 @@
+module Mushy
+
+  class Flux
+
+    attr_accessor :id
+    attr_accessor :parent_fluxs
+    attr_accessor :subscribed_to
+    attr_accessor :config
+    attr_accessor :masher
+
+    def initialize
+      guard
+    end
+
+    class << self
+      attr_accessor :all
+
+      def inherited subclass
+        if (self != Mushy::Flux)
+          Mushy::Flux.inherited subclass
+        else
+          self.all ||= []
+          self.all << subclass
+        end
+      end
+    end
+
+    def guard
+      self.id ||= SecureRandom.uuid
+      self.parent_fluxs ||= []
+      self.subscribed_to ||= []
+      self.masher ||= Masher.new
+      self.config ||= SymbolizedHash.new
+    end
+
+    def execute incoming_event
+      guard
+
+      incoming_event = SymbolizedHash.new(incoming_event) if incoming_event.is_a?(Hash)
+
+      event = incoming_event
+
+      mashed_config = masher.mash config, event
+
+      incoming_split = mashed_config[:incoming_split]
+
+      events = incoming_split ? incoming_event[incoming_split] : [event]
+
+      results = events.map { |e| execute_single_event e, config }
+
+      return results.first unless incoming_split
+
+      results = join_these_results(results, event, config[:join]) if config[:join]
+
+      results.flatten
+    end
+
+    def execute_single_event event, config
+
+      mashed_config = masher.mash config, event
+
+      the_original_join = mashed_config[:join]
+      mashed_config[:join] = nil if mashed_config[:incoming_split]
+
+      results = process event, mashed_config
+
+      returned_one_result = results.is_a?(Hash)
+
+      results = standardize_these results
+      results = shape_these results, event, mashed_config
+
+      return results.first if the_original_join
+
+      return results if mashed_config[:outgoing_split]
+      
+      returned_one_result ? results.first : results
+
+    end
+
+    def standardize_these results
+      [results]
+        .flatten
+        .map { |x| x.is_a?(Hash) ? convert_to_symbolized_hash(x) : nil }
+        .select { |x| x }
+    end
+
+    def shape_these results, event, config
+      supported_shaping = [:merge, :outgoing_split, :group, :model, :join, :sort, :limit]
+
+      shaping = supported_shaping
+      if (config[:shaping])
+        shaping = convert_this_to_an_array(config[:shaping]).map { |x| x.to_sym }
+      end
+
+      supported_shaping
+        .select { |x| config[x] }
+        .each_with_index
+        .sort_by { |x, i| shaping.index(x) || i + supported_shaping.count }
+        .map { |x, _| x }
+        .reduce(results) { |t, i| self.send("#{i}_these_results".to_sym, t, event, config[i]) }
+    end
+
+    def sort_these_results results, event, by
+      results.sort { |x| x[by].to_i }
+    end
+
+    def limit_these_results results, event, by
+      results
+        .each_with_index
+        .select { |x, i| i < by.to_i }
+        .map { |x, _| x }
+    end
+
+    def group_these_results results, event, by
+      group_by = by.split('|')[0]
+      result_key = by.split('|')[1]
+      results.group_by { |x| x[group_by] }.map { |k, v| SymbolizedHash.new( { result_key => v } ) }
+    end
+
+    def outgoing_split_these_results results, event, by
+      results.map { |x| Masher.new.dig by, x }.flatten
+    end
+
+    def join_these_results results, event, by
+      [SymbolizedHash.new( { by => results } )]
+    end
+
+    def model_these_results results, event, by
+      return results unless by.any?
+      results.map { |x| masher.mash by, x }
+    end
+
+    def merge_these_results results, event, by
+      keys_to_merge = convert_this_to_an_array by
+      keys_to_merge = event.keys.map { |x| x.to_s } if (keys_to_merge[0] == '*')
+
+      results.map do |result|
+                    event.select { |k, _| keys_to_merge.include? k.to_s }.each do |k, v|
+                      result[k] = v unless result[k]
+                    end
+                    result
+                  end
+    end
+
+    def convert_this_to_an_array value
+      [value]
+        .flatten
+        .map { |x| x.to_s.split(',').map { |x| x.strip } }
+        .flatten
+        .select { |x| x && x != '' }
+    end
+
+    def convert_to_symbolized_hash event
+      data = SymbolizedHash.new
+      event.each { |k, v| data[k] = v }
+      data
+    end
+
+    def process event, config
+      event
+    end
+
+  end
+
+end

data/lib/mushy/fluxs/bash.rb

@@ -0,0 +1,41 @@
+module Mushy
+  
+  class Bash < Flux
+
+    def self.details
+      {
+        name: 'Bash',
+        description: 'Run a bash command.',
+        config: {
+          command: {
+                     description: 'The command to run in bash.',
+                     type:        'text',
+                     value:       '{{command}}',
+                   },
+          directory: {
+                       description: 'The working directory in which the command will be run.',
+                       type:        'text',
+                       value:       '',
+                     },
+        },
+      }
+    end
+
+    def process event, config
+      command = config[:command]
+
+      command = "cd #{config[:directory]};#{command}" if config[:directory]
+
+      text = `#{command}`
+
+      result = $?
+      {
+        text: text,
+        success: result.success?,
+        exit_code: result.to_i,
+      }
+    end
+
+  end
+
+end

data/lib/mushy/fluxs/browser.rb

@@ -0,0 +1,106 @@
+require 'ferrum'
+
+module Mushy
+
+  class Browser < Flux
+
+    def self.details
+      {
+        name: 'Browser',
+        description: 'Visit a page in a browser.',
+        config: {
+          url: {
+                 description: 'The URL to visit.',
+                 type:        'text',
+                 value:       'https://www.google.com',
+               },
+          headless: {
+                      description: 'Run this browser headless.',
+                      type:        'boolean',
+                      value:       'true',
+                    },
+          execute: {
+                     description: 'Javascript to run after the page is loaded.',
+                     type:        'textarea',
+                     value:       '',
+                   },
+          cookies: {
+                     description: 'Cookies for the web request. These can be received from a previous browser event with {{cookies}}, or can be typed manually.',
+                     type:        'editgrid',
+                     value:       [],
+                     editors: [
+                                 { id: 'name', target: 'name', field: { type: 'text', value: '', default: '' } },
+                                 { id: 'value', target: 'value', field: { type: 'text', value: '', default: '' } },
+                                 { id: 'domain', target: 'domain', field: { type: 'text', value: '', default: '' } },
+                                 { id: 'path', target: 'path', field: { type: 'text', value: '', default: '' } },
+                                 { id: 'expires', target: 'expires', field: { type: 'text', value: '', default: '' } },
+                                 { id: 'size', target: 'size', field: { type: 'integer', value: 0, default: 0 } },
+                                 { id: 'httpOnly', target: 'httpOnly', field: { type: 'boolean', value: false, default: false } },
+                                 { id: 'secure', target: 'secure', field: { type: 'boolean', value: true, default: true } },
+                                 { id: 'sameSite', target: 'sameSite', field: { type: 'text', value: 'None', default: 'None' } },
+                                 { id: 'priority', target: 'priority', field: { type: 'text', value: 'Medium', default: 'Medium' } },
+                              ],
+                     #{"name":"1P_JAR","value":"2021-01-21-13","domain":".google.com","path":"/","expires":1613828458.870408,"size":19,"httpOnly":false,"secure":true,"session":false,"sameSite":"None","priority":"Medium"
+                   },
+          carry_cookies_from: {
+                     description: 'Carry the cookies from this path in the event.',
+                     type:        'text',
+                     value:       'cookies',
+                   },
+          headers: {
+                     description: 'Headers for the web request. These can be received from a previous browser event with {{headers}}, or can be typed manually.',
+                     type:        'keyvalue',
+                     value:       {},
+                   },
+          carry_headers_from: {
+                     description: 'Carry the headers from this path in the event.',
+                     type:        'text',
+                     value:       'headers',
+                   },
+        },
+      }
+    end
+
+    def process event, config
+
+      browser = Ferrum::Browser.new(headless: (config[:headless].to_s != 'false'))
+
+      get_the_cookies_from(event, config).each { |c| browser.cookies.set(c) }
+
+      browser.headers.add get_the_headers_from(event, config)
+
+      browser.goto config[:url]
+
+      browser.execute(config[:execute]) if config[:execute]
+
+      result = {
+        url: browser.url,
+        cookies: browser.cookies.all.map { |k, v| v.instance_variable_get('@attributes') },
+        headers: browser.headers.get,
+        body: browser.body
+      }
+
+      browser.quit
+
+      result
+    end
+
+    def get_the_cookies_from event, config
+      cookies = (event[config[:carry_cookies_from].to_sym])
+      cookies = [] unless cookies.is_a?(Array)
+      config[:cookies] = [] unless config[:cookies].is_a?(Array)
+      config[:cookies].each { |x| cookies << x }
+      cookies
+    end
+
+    def get_the_headers_from event, config
+      headers = (event[config[:carry_headers_from].to_sym])
+      headers = {} unless headers.is_a?(Hash)
+      config[:headers] = {} unless config[:headers].is_a?(Hash)
+      config[:headers].each { |k, v| headers[k] = v }
+      headers
+    end
+
+  end
+
+end

data/lib/mushy/fluxs/build_csv.rb

@@ -0,0 +1,53 @@
+require 'csv'
+
+module Mushy
+
+  class BuildCsv < Flux
+
+    def self.details
+      {
+        name: 'BuildCsv',
+        description: 'Build a CSV.',
+        config: {
+          input_path: {
+                        description: 'The path to the set of records to include in the CSV.',
+                        type:        'text',
+                        value:       'records',
+                      },
+          output_path: {
+                         description: 'The path to the CSV content in the outgoing event.',
+                         type:        'text',
+                         value:       'records',
+                       },
+          headers: {
+                     description: 'The values to include in the CSV, as well as the header values.',
+                     type:        'keyvalue',
+                     value:       {},
+                   },
+          header_row: {
+                        description: 'Include a header row?',
+                        type:        'boolean',
+                        value:       true,
+                      },
+        },
+      }
+    end
+
+    def process event, config
+      records = event[config[:input_path].to_sym] || event[config[:input_path].to_s]
+
+      headers = config[:headers]
+
+      {
+        config[:output_path] => CSV.generate do |c|
+          if config[:header_row].to_s == 'true'
+            c << headers.map { |h| h[1] }
+          end
+          records.each { |x| c << headers.map { |h| x[h[0].to_sym] || x[h[0].to_s] } }
+        end
+      }
+    end
+
+  end
+
+end