logstash-input-google-analytics-daily 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 65e1356004d4e415992c64aa350b9e8f9a32b9cc2caa16ffe8a5cc008e443ce8
-  data.tar.gz: 7f4b32c19012b211a713fe35f0df59a0fad1661dcc90b1c68d8b0bd229afdcbc
+  metadata.gz: 4be2dcf54047afbe18aee1ceac9f2992b8044a1b88cb5d8be5a5fa40ea978ed5
+  data.tar.gz: d5355c716a8c5ee2bfa3b8dd6f5694c8a8151f58f8f6d16fab184b5c09cb3ba6
 SHA512:
-  metadata.gz: 5d79a8193223ef48ccd7c2666a7b23e8bb3237dc71b74e9f4fe5fbaab56a07f30ad3a7292ef82a4bc5cdc51faf20aa8525bb7d3a6ea84a39a4af1adcc9452cc9
-  data.tar.gz: e4bc122fceb88d8c852cc15043f6ca6bb709099a2b4dcc448749215348f668e37f93ace08d4fd42dee092b55dabb9f1f3794f77fe94fd68962e2ffd26239688b
+  metadata.gz: 2b6c4ee7ee1f40e13980d2b0f7b551dbcddc46afd4ed3f234077dedcfbc32bb8819a65db1afdf272811bee25b9869612f690b69616df5e5e0116a33497c2971c
+  data.tar.gz: d10be71488dbb1a07c49da9f0b7cf33530bc27b8b228339201e53e29dd4b8afa21486afc313e4633aa8c8f24022ed889886f4fac06045b1de7038d0fff4124f6

data/README.md

@@ -6,7 +6,7 @@ Forked from https://github.com/russorat/logstash-input-googleanalytics.git.
 
 ## Documentation
 
-Documentation can be seen in [the docs folder](https://github.com/DeimosCloud/logstash-input-google-analytics-daily/blob/master/docs/index.asciidoc). 
+Documentation can be seen in [the docs folder](https://github.com/DeimosCloud/logstash-input-google-analytics-daily/blob/master/docs/index.asciidoc) (for now). Hopefully, this moves to logstash-plugins repo soon, so it can be rendered properly on elastic.co, 
 
 ## Developing
 

data/lib/logstash/inputs/google_analytics_daily.rb

@@ -4,6 +4,7 @@ require "logstash/namespace"
 require "stud/interval"
 require 'google/apis/analytics_v3'
 require 'googleauth'
+require 'json'
 
 # Pull daily reports from Google Analytics using the v3 Core Reporting API.
 # This plugin will generate one Logstash event per date, with each event containing all the data for that date
@@ -26,9 +27,11 @@ class LogStash::Inputs::GoogleAnalyticsDaily < LogStash::Inputs::Base
   # https://developers.google.com/analytics/devguides/reporting/core/v3/reference#ids
   config :view_id, :validate => :string, :required => true
 
-  # This plugin will only fetch reports for the specified dates
-  # In the format YYYY-MM-DD, or relative by using today, yesterday, or the NdaysAgo pattern
-  config :dates, :validate => :string, :list => true, :default => ['yesterday']
+  # This plugin will fetch daily reports for dates in the specified range
+  # In the format YYYY-MM-DD
+  config :start_date, :validate => :string, :required => true
+
+  config :end_date, :validate => :string, :required => true
 
   # The aggregated statistics for user activity to your site, such as clicks or pageviews.
   # Maximum of 10 metrics for any query
@@ -95,7 +98,10 @@ class LogStash::Inputs::GoogleAnalyticsDaily < LogStash::Inputs::Base
 
       analytics = get_service
 
+      @dates = (Date.parse(@start_date)..Date.parse(@end_date))
+
       @dates.each do |date|
+        date = date.to_s
         options = get_request_parameters(date)
 
         results = analytics.get_ga_data(
@@ -155,42 +161,39 @@ class LogStash::Inputs::GoogleAnalyticsDaily < LogStash::Inputs::Base
           query = results.query.to_h
           profile_info = results.profile_info.to_h
 
-          # One event per metric
+          # Transform into proper format for one event per metric
           @metrics.each do |metric|
             rows_for_this_metric = rows.clone.map do |row|
-              # Remove any rows that don't include this metric
-              new_row = row.clone
+              new_row = {}
               new_row[:metric] = row[:metrics].find { |m| m[:name] == metric }
-              new_row.delete(:metrics)
+              new_row[:dimensions] = row[:dimensions]
               new_row
             end
 
-            event = LogStash::Event.new
-            decorate(event)
-            # Populate Logstash event fields
-            event.set('ga.contains_sampled_data', results.contains_sampled_data?)
-            event.set('ga.query', query) if @store_query
-            # We need to convert the date fields here to string because sometimes it's a relative date, so will cause mapping issues
-            event.set('ga.query.start_date', query["start_date"].to_s) if @store_query
-            event.set('ga.query.end_date', query["end_date"].to_s) if @store_query
-
-            event.set('ga.profile_info', profile_info) if @store_profile
-
-            if date == 'today'
-              event.set('ga.date', Time.now.strftime("%F"))
-            elsif date == 'yesterday'
-              event.set('ga.date', Time.at(Time.now.to_i - 86400).strftime("%F"))
-            elsif date.include?('daysAgo')
-              days_ago = date.sub('daysAgo', '').to_i
-              event.set('ga.date', Time.at(Time.now.to_i - (days_ago * 86400)).strftime("%F"))
-            else
+            rows_for_this_metric.each do |row|
+              event = LogStash::Event.new
+              decorate(event)
+              # Populate Logstash event fields
+              event.set('ga.contains_sampled_data', results.contains_sampled_data?)
+              event.set('ga.query', query.to_json) if @store_query
+              event.set('ga.profile_info', profile_info) if @store_profile
               event.set('ga.date', date)
-            end
 
-            event.set("ga.rows", rows_for_this_metric)
-            # Use date and metrics as ID to prevent duplicate entries in Elasticsearch
-            event.set('[@metadata][id]', event.get('ga.date') + metric)
-            queue << event
+              event.set("ga.metric.name", metric)
+              event.set("ga.metric.value", row[:metric][:value])
+
+
+              # Remap dimensions into key: value
+              # Might lead to "mapping explosion", but otherwise aggregations are tough
+              joined_dimension_name = ''
+              row[:dimensions].each do |d|
+                dimension_name = d[:name].sub("ga:", '')
+                joined_dimension_name += dimension_name
+                event.set("ga.dimensions.#{dimension_name}", d[:value])
+              end
+
+              queue << event
+            end
           end
         end
       end

data/logstash-input-google-analytics-daily.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-input-google-analytics-daily'
-  s.version = '0.2.1'
+  s.version = '0.3.0'
   s.licenses = ['Apache License (2.0)']
   s.summary = "Logstash plugin to pull daily reports from Google Analytics."
   s.description = "Logstash plugin to pull daily reports from the Google Analytics v3 Core Reporting API. Install into Logstash using $LS_HOME/bin/logstash-plugin install logstash-input-google-analytics-daily."
@@ -22,7 +22,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'stud', '>= 0.0.22'
   s.add_runtime_dependency 'google-api-client', "~> 0.37"
   s.add_runtime_dependency 'googleauth', ">= 0.10.0"
+  s.add_runtime_dependency 'json'
   s.add_development_dependency 'vcr'
   s.add_development_dependency 'webmock'
-  s.add_development_dependency 'json'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-google-analytics-daily
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Shalvah
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-04-09 00:00:00.000000000 Z
+date: 2020-04-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -78,8 +78,8 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: vcr
-  type: :development
+  name: json
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
@@ -92,7 +92,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: webmock
+  name: vcr
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -106,7 +106,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: json
+  name: webmock
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement