greenhat 0.5.0 → 0.6.2

data/lib/greenhat/shell/process.rb

@@ -30,8 +30,10 @@ module GreenHat
         puts
       end
 
-      def self.filter_help
-        ShellHelper::Filter.help
+      def self.filter_help(raw = {})
+        args, flags, _args = Args.parse(raw)
+
+        ShellHelper.show(ShellHelper::Filter.help(args.first), flags)
       end
 
       def self.ps(raw = {})
@@ -45,6 +47,10 @@ module GreenHat
         ShellHelper.file_output(files, flags)
       end
 
+      def self.default(raw_list)
+        filter(raw_list)
+      end
+
       def self.filter(raw = {})
         # Argument Parsing
         files, flags, args = Args.parse(raw)
@@ -52,14 +58,14 @@ module GreenHat
         # Prepare Log List
         file_list = ShellHelper.prepare_list(files, GreenHat::Ps.things)
 
-        results = ShellHelper.filter_start(file_list, flags, args)
+        results = Query.start(file_list, flags, args)
 
         # Check Search Results
         if results.instance_of?(Hash) && results.values.flatten.empty?
           puts 'No results'.pastel(:red)
         else
-          # This causes the key 'colorized' output to also be included
-          ShellHelper.show(results.to_a.compact.flatten, flags)
+
+          ShellHelper.show(results, flags)
         end
       end
     end

data/lib/greenhat/shell/query.rb

@@ -5,43 +5,133 @@ module GreenHat
   module Query
     # Main Entry Point for Filtering
     def self.start(files, flags = {}, args = {})
-      # Convert to Things
-      files = ShellHelper.find_things(files, flags).select(&:query?)
+      # Convert to Things / Thing.list == processed files
+      files = ShellHelper.find_things(files, flags, Thing.list).select(&:query?)
+
+      # Breakdown by Interval
+      return process_interval(files, flags, args) if flags.key? :interval
 
       # Ignore Archive/Host Dividers
       if flags[:combine]
-        results = files.reject(&:blank?).map(&:data).flatten.compact
-        Query.filter(results, flags, args)
+        # Add SRC Field to Combined Entries
+        results = combine_src_add(files)
+        Query.filter(results.flatten.compact, flags, args)
       else
-        # Iterate and Preserve Archive/Host Index
-        files.each_with_object({}) do |file, obj|
-          # Ignore Empty Results / No Thing
-          next if file&.blank?
+        file_filter(files, flags, args)
+      end
+    end
 
-          # Include Total Count in Name
-          results = Query.filter(file.data, flags, args)
-          duration = calculate_duration(results)
+    def self.combine_src_add(files)
+      files.reject(&:blank?).map do |file|
+        src = file.archive&.friendly_name || file&.name
+        file.data.map do |entry|
+          bit = entry.clone
+          bit[:src] = src
+          bit
+        end
+      end
+    end
 
-          title = [
-            file.friendly_name,
-            " #{results.count}".pastel(:bright_black)
+    # Simplify Loop for Query.start
+    def self.file_filter(files, flags = {}, args = {})
+      # Iterate and Preserve Archive/Host Index
+      files.each_with_object([]) do |file, obj|
+        # Ignore Empty Results / No Thing
+        next if file&.blank?
 
-          ]
+        # Include Total Count in Name
+        results = Query.filter(file.data, flags, args)
 
-          # Append Duration
-          title.push(" #{duration.pastel(:cyan, :dim)}") unless duration.blank?
+        next if results.count.zero? # Skip if there are no results
 
-          # Save unless empty
-          obj[title.join] = results unless results.count.zero?
+        duration = calculate_duration(results)
 
-          obj
-        end
+        # Create Title
+        title = create_title(file, flags, results)
+
+        # Append Duration
+        title += " #{duration.pastel(:cyan, :dim)}" unless duration.blank?
+
+        # Add Title
+        obj.push(title)
+
+        # Add Results
+        obj.concat(results)
+
+        obj
+      end
+    end
+
+    # Helper to simplify Title Creation
+    def self.create_title(file, flags, results)
+      title = file.friendly_name
+      # Ignore for Total Results
+      title += " #{results.count}".pastel(:bright_black) unless flags.key?(:total)
+      title
+    end
+
+    # Helper to transform fields from one to another (timestamp => time)
+    def self.process_transform(transform, results)
+      # TODO: Eventually provide more splat options
+      from, to, *splat = transform
+      results.each do |entry|
+        entry[to.to_sym] = case splat
+                           when [:to_i]
+                             entry[from.to_sym].to_i
+                           else
+                             entry[from.to_sym]
+                           end
+      end
+    end
+
+    # Organize Results by Interval
+    # TODO Simplify
+    # rubocop:disable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
+    def self.process_interval(files, flags, args)
+      data = files.reject(&:blank?).map(&:data).flatten.compact
+
+      # Handle Transform
+      process_transform(flags[:transform], data) if flags.key? :transform
+
+      interval = ChronicDuration.parse(flags[:interval]) || 5.minutes
+      index = build_time_index(data, interval)
+      index.transform_values! { |_y| [] }
+
+      data.each do |r|
+        index[r.time.floor(interval)] << r
+      end
+
+      output = index.each_with_object({}) do |(time, entries), obj|
+        results = Query.filter(entries, flags, args)
+
+        next if results.count.zero? # Skip if No Results
+
+        duration = calculate_duration(results)
+
+        # Timzone Manipulation
+        time_title = flags.key?(:time_zone) ? time.in_time_zone(flags[:time_zone]) : time
+
+        title = time_title.to_s.pastel(:bright_blue, :bold)
+        title += " #{results.count}".pastel(:bright_black) unless flags.key?(:total)
+
+        # Append Duration
+        title += " #{duration.pastel(:cyan, :dim)}" unless duration.blank?
+
+        # Add Results
+        obj[title] = results
+
+        obj
       end
+
+      # Remove Empty Intervals / There may be empty here due to results
+      output.reject! { |_k, v| v.flatten.empty? }
+
+      output.to_a.flatten(2)
     end
 
     def self.calculate_duration(results)
       # Skip for Pluck
-      only_with_time = results.select { |x| x.instance_of?(Hash) && x.key?(:time) }
+      only_with_time = results.select { |x| x.instance_of?(Hash) && x.key?(:time) && x[:time].instance_of?(Time) }
 
       # If slice is used ignore
       return nil if only_with_time.empty?
@@ -67,12 +157,11 @@ module GreenHat
 
     # Filter Logic
     # TODO: Simplify
-    # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
     def self.filter(data, flags = {}, args = {})
-      # Experimenting with deep clone
-      # results = Marshal.load(Marshal.dump(data))
       results = data.clone
-      # results = data
+
+      # Handle Transform
+      process_transform(flags[:transform], results) if flags.key? :transform
 
       results.select! do |row|
         args.send(flags.logic || :all?) do |arg|
@@ -98,8 +187,11 @@ module GreenHat
       # Remove Blank from either slice or except
       results.reject!(&:empty?)
 
-      # Sort
-      results.sort_by! { |x| x.slice(*flags[:sort]).values } if flags.key?(:sort)
+      # Sort / Reverse by default
+      if flags.key?(:sort)
+        results.sort_by! { |x| x.slice(*flags[:sort]).values }
+        results.reverse!
+      end
 
       # JSON Formatting
       results = results.map { |x| Oj.dump(x) } if flags.key?(:json)
@@ -113,16 +205,41 @@ module GreenHat
       # Count occurrences / Skip Results
       return filter_stats(results, flags) if flags.key?(:stats)
 
+      # Percentile Breakdown
+      return filter_percentile(results, flags) if flags.key?(:percentile)
+
       # Limit before Pluck / Flattening
       results = filter_limit(results, flags[:limit]) if flags.key?(:limit)
 
       # Pluck
       results = filter_pluck(results, flags[:pluck]) if flags.key?(:pluck)
 
+      # Total Counter
+      return [ShellHelper.total_count(results, flags)] if flags.key?(:total)
+
       results
     end
     # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
 
+    # Create list of Times
+    def self.build_time_index(results, interval = 5.minutes)
+      index = {}
+      start = results.min_by(&:time).time.floor(interval)
+      finish = results.max_by(&:time).time.floor(interval)
+
+      loop do
+        index[start] = 0
+        start += interval
+        break if start > finish
+      end
+
+      index
+    rescue StandardError => e
+      LogBot.fatal('Index Error', e.message)
+    ensure
+      {}
+    end
+
     # Limit / Ensure Exists and Valid Number
     def self.filter_limit(results, limit)
       return results unless limit.integer? && limit.positive?
@@ -231,6 +348,45 @@ module GreenHat
       end.inject(:&)
     end
 
+    # TODO: Make better
+    # rubocop:disable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
+    def self.filter_percentile(results, flags)
+      output = {}
+
+      results.each do |entry|
+        entry.each do |key, value|
+          # Numbers only
+          next unless value.instance_of?(Integer) || value.instance_of?(Float)
+
+          output[key] ||= []
+          output[key].push value
+        end
+      end
+
+      data = output.map do |key, values|
+        # Support Rounding
+        l99 = values.percentile(0.99)
+        l99 = l99.round(flags.round) if flags.round
+
+        l95 = values.percentile(0.99)
+        l95 = l95.round(flags.round) if flags.round
+
+        {
+          key: key,
+          '99' => l99,
+          '95' => l95,
+          mean: flags.round ? values.mean.round(flags.round) : values.mean,
+          min: flags.round ? values.min.round(flags.round) : values.min,
+          max: flags.round ? values.max.round(flags.round) : values.max,
+          count: values.count
+        }
+      end
+
+      headers = data.flat_map(&:keys).uniq
+      [[headers, data.map(&:values)]] # Multiple Arrays for Results Flatten
+    end
+    # rubocop:enable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
+
     def self.filter_stats(results, flags)
       stats = flags[:stats]