log_sense 1.9.0 → 2.0.0

data/lib/log_sense/apache_report_shaper.rb

@@ -64,38 +64,12 @@ module LogSense
           }
         ",
         },
-        {
-          title: "Time Distribution",
+        time_distribution(
+          data,
           header: %w[Hour Hits Visits Size],
           column_alignment: %i[left right right right],
-          rows: data[:time_distribution],
-          echarts_spec: "{
-            xAxis: {
-              type: 'category',
-              data: SERIES_DATA.map(row => row['Hour'])
-              /* data: ['00', '01', '02', '03', '04', '05', '06', '07', '08', '09',
-                     '10', '11', '12', '13', '14', '15', '16', '17', '18', '19',
-                     '20', '21', '22', '23', '24'] */
-            },
-            yAxis: {
-              type: 'value'
-            },
-            tooltip: {
-               trigger: 'axis'
-            },
-            series: [
-              {
-                data: SERIES_DATA.map(row => row['Hits']),
-                type: 'bar',
-                color: '#4C78A8',
-                label: {
-                   show: true,
-                   position: 'top'
-                },
-              }
-            ]
-          }",
-        },
+          color: "#4C78A8"
+        ),
         {
           title: "20_ and 30_ on HTML pages",
           header: %w[Path Hits Visits Size Status],
@@ -154,242 +128,27 @@ module LogSense
             ]
           }&.sort { |x, y| y[1] <=> x[1] }
         },
-        {
-          title: "Statuses",
-          header: %w[Status Count],
-          column_alignment: %i[left right],
-          rows: data[:statuses],
-          echarts_spec: "{
-            xAxis: {
-              type: 'category',
-              data: SERIES_DATA.map(row => row['Status'])
-              /* data: ['100', '101', '102', '103',
-                     '200', '201', '202', '203', '204', '205', '206', '207', '208', '226',
-                     '300', '301', '302', '303', '304', '305', '306', '307', '308',
-                     '400', '401', '402', '403', '404', '405', '406', '407', '408', '409', '410', '411', '412', '413', '414', '415', '416', '417', '418', '421', '422', '423', '424', '425', '426', '428', '429', '431', '451',
-                     '500', '501', '502', '503', '504', '505', '506', '507', '508', '510', '511'] */
-            },
-            yAxis: {
-              type: 'value'
-            },
-            tooltip: {
-               trigger: 'axis'
-            },
-            series: [
-              {
-                data: SERIES_DATA.map(row => row['Count']),
-                type: 'bar',
-                color: '#4C78A8',
-                label: {
-                   show: true,
-                   position: 'top'
-                },
-              }
-            ]
-          }",
-        },
-        {
-          title: "Daily Statuses",
-          header: %w[Date S_2xx S_3xx S_4xx S_5xx],
-          column_alignment: %i[left right right right right],
-          rows: data[:statuses_by_day],
-          echarts_spec: "{
-            xAxis: {
-              type: 'category',
-              data: SERIES_DATA.map(row => row['Date'])
-            },
-            yAxis: {
-              type: 'value'
-            },
-            tooltip: {
-               trigger: 'axis'
-            },
-            series: [
-              {
-                data: SERIES_DATA.map(row => row['S_2xx']),
-                type: 'bar',
-                color: '#218521',
-                stack: 'total',
-                label: {
-                   show: true,
-                   position: 'right'
-                },
-              },
-              {
-                data: SERIES_DATA.map(row => row['S_3xx']),
-                type: 'bar',
-                color: '#FF8C00',
-                stack: 'total',
-                label: {
-                   show: true,
-                   position: 'right'
-                },
-              },
-              {
-                data: SERIES_DATA.map(row => row['S_4xx']),
-                type: 'bar',
-                color: '#A52A2A',
-                stack: 'total',
-                label: {
-                   show: true,
-                   position: 'right'
-                },
-              },
-              {
-                data: SERIES_DATA.map(row => row['S_5xx']),
-                type: 'bar',
-                color: '#000000',
-                stack: 'total',
-                label: {
-                   show: true,
-                   position: 'right'
-                },
-              },
-            ]
-          }",
-        },
-        {
-          title: "Browsers",
+        total_statuses(data),
+        daily_statuses(data),
+        browsers(
+          data,
           header: %w[Browser Hits Visits Size],
           column_alignment: %i[left right right right],
-          rows: data[:browsers],
-          echarts_spec: "{
-            toolbox: {
-               feature: {
-                 saveAsImage: {},
-               }
-            },
-            tooltip: {
-               trigger: 'axis'
-            },
-            xAxis: {
-              type: 'category',
-              data: SERIES_DATA.sort(order_by_name).map(row => row['Browser']),
-              showGrid: true,
-              axisLabel: {
-                rotate: 45 // Rotate the labels by 90 degrees
-              }
-            },
-            yAxis: {
-              type: 'value',
-              name: 'Browser Hits',
-              showGrid: true,
-            },
-            series: [
-              {
-                name: 'Hits',
-                data: SERIES_DATA.sort(order_by_name).map(row => row['Hits']),
-                type: 'bar',
-                color: '#4C78A8',
-                label: {
-                  show: true,
-                  position: 'top'
-                },
-              },
-            ]
-          }
-          function order_by_name(a, b) {
-            return a['Browser'] < b['Browser'] ? -1 : 1
-          }
-          ",
-        },
-        {
-          title: "Platforms",
+          color: '#4C78A8'
+        ),
+        platforms(
+          data,
           header: %w[Platform Hits Visits Size],
           column_alignment: %i[left right right right],
-          rows: data[:platforms],
-          echarts_spec: "{
-            toolbox: {
-               feature: {
-                 saveAsImage: {},
-               }
-            },
-            tooltip: {
-               trigger: 'axis'
-            },
-            xAxis: {
-              type: 'category',
-              data: SERIES_DATA.sort(order_by_platform).map(row => row['Platform']),
-              showGrid: true,
-              axisLabel: {
-                rotate: 45 // Rotate the labels by 90 degrees
-              }
-            },
-            yAxis: {
-              type: 'value',
-              name: 'Platform Hits',
-              showGrid: true,
-            },
-            series: [
-              {
-                name: 'Hits',
-                data: SERIES_DATA.sort(order_by_platform).map(row => row['Hits']),
-                type: 'bar',
-                color: '#4C78A8',
-                label: {
-                  show: true,
-                  position: 'top'
-                },
-              },
-            ]
-          }
-          function order_by_platform(a, b) {
-            return a['Platform'] < b['Platform'] ? -1 : 1
-          }",
-        },
-        {
-          title: "IPs",
+          color: '#4C78A8'
+        ),
+        ips(
+          data,
           header: %w[IP Hits Visits Size Country],
           column_alignment: %i[left right right right left],
-          rows: data[:ips]
-        },
-        {
-          title: "Countries",
-          header: ["Country", "Hits", "Visits", "IPs", "IP List"],
-          column_alignment: %i[left right right right left],
-          rows: data[:countries]&.map { |k, v|
-            [
-              k,
-              v.map { |x| x[1] }.inject(&:+),
-              v.map { |x| x[2] }.inject(&:+),
-              v.map { |x| x[0] }.uniq.size,
-              v.map { |x| x[0] }.join(WORDS_SEPARATOR)
-            ]
-          }&.sort { |x, y| y[3] <=> x[3] },
-          echarts_height: "600px",
-          echarts_spec: "{
-            tooltip: {
-                trigger: 'axis',
-                axisPointer: {
-                  type: 'shadow'
-                }
-            },
-            xAxis: {
-              type: 'value',
-              boundaryGap: [0, 0.01]
-            },
-            yAxis: {
-              type: 'category',
-              data: SERIES_DATA.sort(order_by_hits).map(row => row['Country'] ),
-            },
-            series: [
-              {
-                 type: 'bar',
-                 data: SERIES_DATA.sort(order_by_hits).map(row => row['Hits'] ),
-                 color: '#4C78A8',
-                 label: {
-                    show: true,
-                    position: 'right'
-                 },
-              },
-            ]
-          };
-
-          function order_by_hits(a, b) {
-            return Number(a['Hits']) < Number(b['Hits']) ? -1 : 1
-          }
-          "
-        },
+          palette: :apache
+        ),
+        countries(data, color: '#4C78A8'),
         ip_per_hour_report_spec(ips_per_hour(data[:ips_per_hour])),
         {
           title: "Combined Platform Data",

data/lib/log_sense/emitter.rb

@@ -28,6 +28,8 @@ module LogSense
       @data = data
       @options = options
       @report_title = options[:input_format].capitalize
+
+      @format_specific_theme = "#{@options[:input_format]}_theme.css"
       @format_specific_css = "#{@options[:input_format]}.css.erb"
 
       # Chooses template and destination

data/lib/log_sense/rails_aggregator.rb

@@ -122,15 +122,25 @@ module LogSense
                 ON event.log_id == error.log_id
                 WHERE #{filter} and substr(status, 1, 1) == '5').gsub("\n", "") || [[]]
 
+      @internal_server_error_plot = @db.execute %Q(
+         SELECT strftime('%Y-%m-%d', started_at) as Day,
+                count(distinct(event.id)) as Errors
+                FROM Event JOIN Error
+                ON event.log_id == error.log_id
+                WHERE #{filter} and substr(status, 1, 1) == '5'
+                GROUP BY strftime('%Y-%m-%d', started_at)).gsub("\n", "") || [[]]
+
       @error = @db.execute %Q(
-         SELECT filename, log_id, description, count(log_id)
+         SELECT filename,
+                log_id, description, count(log_id)
                 FROM Error
                 WHERE (description NOT LIKE '%No route matches%' and
                        description NOT LIKE '%Couldn''t find%')
                 GROUP BY description).gsub("\n", "") || [[]]
       
       @possible_attacks = @db.execute %Q(
-         SELECT filename, log_id, description, count(log_id)
+         SELECT filename,
+                log_id, description, count(log_id)
                 FROM Error
                 WHERE (description LIKE '%No route matches%' or
                        description LIKE '%Couldn''t find%')

data/lib/log_sense/rails_report_shaper.rb

@@ -39,73 +39,9 @@ module LogSense
             ]
           };",
         },
-        {
-          title: "Time Distribution",
-          header: %w[Hour Hits],
-          column_alignment: %i[left right],
-          rows: data[:time_distribution],
-          echarts_spec: "{
-            xAxis: {
-              type: 'category',
-              data: SERIES_DATA.map(row => row['Hour'])
-              /* data: ['00', '01', '02', '03', '04', '05', '06', '07', '08', '09',
-                     '10', '11', '12', '13', '14', '15', '16', '17', '18', '19',
-                     '20', '21', '22', '23', '24'] */
-            },
-            yAxis: {
-              type: 'value'
-            },
-            tooltip: {
-               trigger: 'axis'
-            },
-            series: [
-              {
-                data: SERIES_DATA.map(row => row['Hits']),
-                type: 'bar',
-                color: '#D30001',
-                label: {
-                   show: true,
-                   position: 'top'
-                },
-              }
-            ]
-          }",
-        },
-        {
-          title: "Statuses",
-          header: %w[Status Count],
-          column_alignment: %i[left right],
-          rows: data[:statuses],
-          col: "small-12 cell",
-          echarts_spec: "{
-            xAxis: {
-              type: 'category',
-              data: SERIES_DATA.map(row => row['Status'])
-              /* data: ['100', '101', '102', '103',
-                     '200', '201', '202', '203', '204', '205', '206', '207', '208', '226',
-                     '300', '301', '302', '303', '304', '305', '306', '307', '308',
-                     '400', '401', '402', '403', '404', '405', '406', '407', '408', '409', '410', '411', '412', '413', '414', '415', '416', '417', '418', '421', '422', '423', '424', '425', '426', '428', '429', '431', '451',
-                     '500', '501', '502', '503', '504', '505', '506', '507', '508', '510', '511'] */
-            },
-            yAxis: {
-              type: 'value'
-            },
-            tooltip: {
-               trigger: 'axis'
-            },
-            series: [
-              {
-                data: SERIES_DATA.map(row => row['Count']),
-                type: 'bar',
-                color: '#D30001',
-                label: {
-                   show: true,
-                   position: 'top'
-                },
-              }
-            ]
-          }",
-        },
+        time_distribution(data),
+        total_statuses(data),
+        daily_statuses(data),
         {
           title: "Rails Performance",
           header: %w[Controller Hits Min Avg Max],
@@ -208,7 +144,7 @@ module LogSense
               data: fatal_plot.filter(row => row[0] != '').map(row => row[0]),
               showGrid: true,
               axisLabel: {
-                rotate: 45 // Rotate the labels by 90 degrees
+                rotate: 45 // Rotate the labels
               }
             },
             yAxis: {
@@ -234,72 +170,8 @@ module LogSense
           header: %w[Date Status IP URL Description Log ID],
           column_alignment: %i[left left left left left left],
           rows: data[:internal_server_error],
-          col: "small-12 cell"
-        },
-        {
-          title: "Errors",
-          header: %w[Log ID Description Count],
-          column_alignment: %i[left left left],
-          rows: data[:error],
-          col: "large-6 cell"
-        },
-        {
-          title: "Possible Attacks",
-          header: %w[Log ID Description Count],
-          column_alignment: %i[left left left],
-          rows: data[:possible_attacks],
-          col: "large-6 cell"
-        },
-        {
-          title: "Browsers",
-          header: %w[Browser Visits],
-          column_alignment: %i[left right],
-          rows: data[:browsers],
-          echarts_spec: "{
-            toolbox: {
-               feature: {
-                 saveAsImage: {},
-               }
-            },
-            tooltip: {
-               trigger: 'axis'
-            },
-            xAxis: {
-              type: 'category',
-              data: SERIES_DATA.sort(order_by_name).map(row => row['Browser']),
-              showGrid: true,
-              axisLabel: {
-                rotate: 45 // Rotate the labels by 90 degrees
-              }
-            },
-            yAxis: {
-              type: 'value',
-              name: 'Browser Visits',
-              showGrid: true,
-            },
-            series: [
-              {
-                name: 'Hits',
-                data: SERIES_DATA.sort(order_by_name).map(row => row['Visits']),
-                type: 'bar',
-                color: '#D30001',
-                label: {
-                  show: true,
-                  position: 'top'
-                },
-              },
-            ]
-          }
-          function order_by_name(a, b) {
-            return a['Browser'] < b['Browser'] ? -1 : 1
-          }
-          ",
-        },
-        {
-          title: "Platforms",
-          header: %w[Platform Visits],
-          column_alignment: %i[left right],
-          rows: data[:platforms],
+          col: "small-12 cell",
+          echarts_extra: "var internal_server_error_plot=#{data[:internal_server_error_plot].to_json}",
           echarts_spec: "{
             toolbox: {
                feature: {
@@ -311,21 +183,20 @@ module LogSense
             },
             xAxis: {
               type: 'category',
-              data: SERIES_DATA.sort(order_by_platform).map(row => row['Platform']),
+              data: internal_server_error_plot.filter(row => row[0] != '').map(row => row[0]),
               showGrid: true,
               axisLabel: {
-                rotate: 45 // Rotate the labels by 90 degrees
+                rotate: 45 // Rotate the labels
               }
             },
             yAxis: {
               type: 'value',
-              name: 'Platform Visits',
+              name: 'Errors',
               showGrid: true,
             },
             series: [
               {
-                name: 'Visits',
-                data: SERIES_DATA.sort(order_by_platform).map(row => row['Visits']),
+                data: internal_server_error_plot.filter(row => row[0] != '').map(row => row[1]),
                 type: 'bar',
                 color: '#D30001',
                 label: {
@@ -334,73 +205,29 @@ module LogSense
                 },
               },
             ]
-          }
-          function order_by_platform(a, b) {
-            return a['Platform'] < b['Platform'] ? -1 : 1
-          }",
+          };"
         },
         {
-          title: "IPs",
-          header: %w[IPs Hits Country],
-          column_alignment: %i[left right left],
-          rows: data[:ips]
+          title: "Errors",
+          header: %w[Log ID Description Count],
+          column_alignment: %i[left left left right],
+          rows: data[:error],
+          col: "small-12 cell"
         },
         {
-          title: "Countries",
-          header: ["Country", "Hits", "IPs", "IP List"],
-          column_alignment: %i[left right left],
-          rows: countries_table(data[:countries]),
-          echarts_height: "600px",
-          echarts_spec: "{
-            tooltip: {
-                trigger: 'axis',
-                axisPointer: {
-                  type: 'shadow'
-                }
-            },
-            xAxis: {
-              type: 'value',
-              boundaryGap: [0, 0.01]
-            },
-            yAxis: {
-              type: 'category',
-              data: SERIES_DATA.sort(order_by_hits).map(row => row['Country'] ),
-            },
-            series: [
-              {
-                 type: 'bar',
-                 data: SERIES_DATA.sort(order_by_hits).map(row => row['Hits'] ),
-                 color: '#D30001',
-                 label: {
-                    show: true,
-                    position: 'right'
-                 },
-              },
-            ]
-          };
-
-          function order_by_hits(a, b) {
-            return Number(a['Hits']) < Number(b['Hits']) ? -1 : 1
-          }
-          "
+          title: "Potential Attacks",
+          header: %w[Log ID Description Count],
+          column_alignment: %i[left left left right],
+          rows: data[:possible_attacks],
+          col: "small-12 cell"
         },
+        browsers(data),
+        platforms(data),
+        ips(data),
+        countries(data),
         ip_per_hour_report_spec(ips_per_hour(data[:ips_per_hour])),
         session_report_spec(ips_detailed(data[:ips_per_day_detailed]))
       ]
     end
-
-    private
-
-    # { country => [[ip, visit, country], ...]
-    def countries_table(data)
-      data&.map { |k, v|
-        [
-          k || "-",
-          v.map { |x| x[1] }.inject(&:+),
-          v.map { |x| x[0] }.uniq.size,
-          v.map { |x| x[0] }.join(WORDS_SEPARATOR)
-        ]
-      }&.sort { |x, y| x[0] <=> y[0] }
-    end
   end
 end