logstash-input-google-analytics-daily 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 1acf86c6fbba654cc2b169ba1a37b37a5e3c7287f95dd31b0b0cdea4eb5b278c
+  data.tar.gz: 967c8c040e7e02a1e7293817e24837a71fdad9530ca629a168f6a0810cc1250e
+SHA512:
+  metadata.gz: 8ef569d8ed0f2cec8365c6ff7acbc98aaeba8662159d611962db63eb3ef5b6661991cb8c09923106635385384e76f49f30995f9c166a664e4770f56f413b4650
+  data.tar.gz: d3b9e1e1bcad5ad20b7bf57f7152c395a2dca7eafea0fc12ad2bf667262383f4d8a56d22f9a613948a5b9980f59be1fdbf10a4727b734f72c2ed2bb220dd5b89

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+## 1.0.0
+ - Initial Release

data/DEVELOPER.md

@@ -0,0 +1,2 @@
+# logstash-input-google-analytics-daily
+Use this input to query data from Google Analytics into Logstash

data/Gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+gemspec
+
+# Needed this for testing, as Logstash dependency management is currently broken
+#
+# logstash_path = ENV["LOGSTASH_PATH"] || "C:/logstash-7.6.1"
+# gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+# gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/NOTICE.TXT

@@ -0,0 +1,5 @@
+Elasticsearch
+Copyright 2012-2015 Elasticsearch
+
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).

data/README.md

@@ -0,0 +1,89 @@
+# Logstash Plugin
+
+Logstash plugin to pull daily reports from the Google Analytics v3 Core Reporting API.
+
+Forked from https://github.com/russorat/logstash-input-googleanalytics.git.
+
+## Documentation
+
+Documentation automatically generated by Logstash at 
+
+## Developing
+
+### 1. Plugin Development and Testing
+
+#### Code
+- To get started, you'll need JRuby with the Bundler gem installed (`jruby -S gem install bundler`).
+
+- Install dependencies
+
+```sh
+jruby -S bundle install
+```
+
+#### Test
+
+- Run tests
+
+```sh
+jruby -S bundle exec rspec
+```
+
+### Running your plugin locally in Logstash
+
+#### Linking the gem
+
+- Edit Logstash's `Gemfile` and add the local plugin path, for example:
+
+```ruby
+gem "logstash-input-google-analytics-daily", :path => "/your/local/logstash-input-googleanalytics"
+```
+
+- Install the plugin
+```sh
+bin/plugin install --no-verify
+```
+- Run Logstash with your plugin
+Example logstash.conf:
+
+```
+input {
+  google_analytics_daily {
+    view_id => "ga:62549480"
+    metrics =>  ["ga:users","ga:sessions"]
+    key_file_path => "C:/logstash-7.6.1/keyfile.json"
+    dates => ['yesterday', '2020-04-05']
+    dimensions => ['ga:browser', 'ga:city']
+  }
+}
+
+output {
+  stdout {
+    codec => rubydebug
+  }
+}
+```
+
+```sh
+bin/logstash -f logstash.conf
+```
+
+NOTE: Any modifications to the plugin code will not show up until you restart Logstash.
+
+#### 2.2 Run in an installed Logstash
+
+ Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using:
+
+- Build your plugin gem
+```sh
+jruby -S gem build logstash-input-google-analytics-daily.gemspec
+```
+- Install the plugin from the Logstash home
+```sh
+bin/plugin install  install Projects/logstash-input-google-analytics-daily/logstash-input-google-analytics-daily-1.0.0.gem
+```
+- Start Logstash with your pipeline config:
+
+```sh
+bin/logstash -f logstash.conf
+```

data/lib/logstash/inputs/google_analytics_daily.rb

@@ -0,0 +1,243 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "stud/interval"
+require 'google/apis/analytics_v3'
+require 'googleauth'
+
+# Pull daily reports from Google Analytics using the v3 Core Reporting API.
+# This plugin will generate one Logstash event per date, with each event containing all the data for that date
+# The plugin will try to maintain a single event per date and list of metrics
+
+class LogStash::Inputs::GoogleAnalyticsDaily < LogStash::Inputs::Base
+  config_name "google_analytics_daily"
+
+  # If undefined, Logstash will complain, even if codec is unused.
+  default :codec, "plain"
+
+  # Most of these inputs are described in the Google Analytics API docs.
+  # https://developers.google.com/analytics/devguides/reporting/core/v3/reference#q_details
+  # Any changes from the format described above have been noted.
+
+  # Type for logstash filtering
+  config :type, :validate => :string, :default => 'google_analytics_daily'
+
+  # View (profile) id, in the format 'ga:XXXX'
+  # https://developers.google.com/analytics/devguides/reporting/core/v3/reference#ids
+  config :view_id, :validate => :string, :required => true
+
+  # This plugin will only fetch reports for the specified dates
+  # In the format YYYY-MM-DD, or relative by using today, yesterday, or the NdaysAgo pattern
+  config :dates, :validate => :string, :list => true, :default => ['yesterday']
+
+  # The aggregated statistics for user activity to your site, such as clicks or pageviews.
+  # Maximum of 10 metrics for any query
+  # https://developers.google.com/analytics/devguides/reporting/core/v3/reference#metrics
+  # For a full list of metrics, see the documentation
+  # https://developers.google.com/analytics/devguides/reporting/core/dimsmets
+  config :metrics, :validate => :string, :list => true, :required => true
+
+  # Breaks down metrics by common criteria; for example, by ga:browser or ga:city
+  # Maximum of 7 dimensions in any query
+  # https://developers.google.com/analytics/devguides/reporting/core/v3/reference#dimensions
+  # For a full list of dimensions, see the documentation
+  # https://developers.google.com/analytics/devguides/reporting/core/dimsmets
+  config :dimensions, :validate => :string, :list => true, :default => []
+
+  # Used to restrict the data returned from your request
+  # https://developers.google.com/analytics/devguides/reporting/core/v3/reference#filters
+  config :filters, :validate => :string, :default => nil
+
+  # A list of metrics and dimensions indicating the sorting order and sorting direction for the returned data
+  # https://developers.google.com/analytics/devguides/reporting/core/v3/reference#sort
+  config :sort, :validate => :string, :default => nil
+
+  # https://developers.google.com/analytics/devguides/reporting/core/v3/reference#segment
+  config :segment, :validate => :string, :default => nil
+
+  # Valid values are DEFAULT, FASTER, HIGHER_PRECISION
+  # https://developers.google.com/analytics/devguides/reporting/core/v3/reference#samplingLevel
+  config :sampling_level, :validate => :string, :default => nil
+
+  # https://developers.google.com/analytics/devguides/reporting/core/v3/reference#include-empty-rows
+  config :include_empty_rows, :validate => :boolean, :default => true
+
+  # These values need to be pulled from your Google Developers Console
+  # For more information, see the docs. Be sure to enable Google Analytics API
+  # access for your application.
+  # https://developers.google.com/identity/protocols/OAuth2ServiceAccount
+
+  # This should be the path to the public/private key as a standard P12 file
+  config :key_file_path, :validate => :string, :required => true
+
+  # This will store the query in the resulting logstash event
+  # https://developers.google.com/analytics/devguides/reporting/core/v3/reference#data_response
+  config :store_query, :validate => :boolean, :default => true
+
+  # This will store the profile information in the resulting logstash event
+  # https://developers.google.com/analytics/devguides/reporting/core/v3/reference#data_response
+  config :store_profile, :validate => :boolean, :default => true
+
+  # Interval to run the command. Value is in seconds. If no interval is given,
+  # this plugin only fetches data once.
+  config :interval, :validate => :number, :required => false, :default => 60 * 60 * 24 # Daily
+
+
+  public
+
+  def register
+  end
+
+  def run(queue)
+    # we abort the loop if stop? becomes true
+    while !stop?
+      start_time = Time.now
+
+      analytics = get_service
+
+      @dates.each do |date|
+        options = get_request_parameters(date)
+
+        results = analytics.get_ga_data(
+            options[:view_id],
+            options[:start_date],
+            options[:end_date],
+            options[:metrics],
+            dimensions: options[:dimensions],
+            filters: options[:filters],
+            include_empty_rows: options[:include_empty_rows],
+            sampling_level: options[:sampling_level],
+            segment: options[:segment],
+            sort: options[:sort],
+        )
+
+        query = results.query.to_h
+        profile_info = results.profile_info.to_h
+        column_headers = results.column_headers.map { |c| c.name }
+
+        event = LogStash::Event.new
+        decorate(event)
+        # Populate Logstash event fields
+        event.set('ga.contains_sampled_data', results.contains_sampled_data?)
+        event.set('ga.query', query) if @store_query
+        event.set('ga.profile_info', profile_info) if @store_profile
+
+        if date == 'today'
+          event.set('ga.date', Time.now.strftime("%F"))
+        elsif date == 'yesterday'
+          event.set('ga.date', Time.at(Time.now.to_i - 86400).strftime("%F"))
+        elsif date.include?('daysAgo')
+          days_ago = date.sub('daysAgo', '').to_i
+          event.set('ga.date', Time.at(Time.now.to_i - (days_ago * 86400)).strftime("%F"))
+        else
+          event.set('ga.date', date)
+        end
+
+        # Use date and metrics as ID to prevent duplicate entries in Elasticsearch
+        event.set('_id', event.get('ga.date') + options[:metrics])
+
+        rows = []
+
+        if results.rows && results.rows.first
+
+          # Example with dimensions, multiple metrics:
+          # rows: [[Chrome, Cape Town, 6, 8], [Chrome, Paris, 1, 5], [Safari, Paris, 1, 3]], column_headers: ['ga:browser', 'ga:city', 'ga:user', 'ga:sessions']
+          # Example with dimension, single metric:
+          # rows: [[Chrome, 6]], column_headers: ['ga:browser', 'ga:user']
+          # Example with no dimension, single metric:
+          # rows: [[6]], column_headers: ['ga:user']
+          # Dimensions always appear before values
+          results.rows.each do |row|
+            dimensions = []
+            metrics = []
+
+            column_headers.zip(row) do |header, value|
+              # Combine GA column headers with values from row
+              if is_num(value)
+                float_value = Float(value)
+                # Sometimes GA returns infinity. if so, the number is invalid
+                # so set it to zero.
+                value = (float_value == Float::INFINITY) ? 0.0 : float_value
+              end
+
+              entry = {
+                  name: header,
+                  value: value
+              }
+              if @metrics.include?(header)
+                metrics << entry
+              else
+                dimensions << entry
+              end
+
+            end
+
+            rows << {metrics: metrics, dimensions: dimensions}
+          end
+
+        end
+        event.set("ga.rows", rows)
+
+        queue << event
+      end
+
+      # If no interval was set, we're done
+      if @interval.nil?
+        break
+      else
+        # Otherwise we sleep till the next run
+        time_lapsed = Time.now - start_time
+        # Sleep for the remainder of the interval, or 0 if the duration ran
+        # longer than the interval.
+        time_to_sleep_for = [0, @interval - time_lapsed].max
+        if time_to_sleep_for == 0
+          @logger.warn(
+              "Execution ran longer than the interval. Skipping sleep.",
+              :duration => time_lapsed,
+              :interval => @interval
+          )
+        else
+          Stud.stoppable_sleep(time_to_sleep_for) { stop? }
+        end
+      end
+    end # loop
+  end
+
+  private
+
+  def get_request_parameters(date)
+    options = {
+        :view_id => @view_id,
+        :start_date => date,
+        :end_date => date,
+        :metrics => @metrics.sort.join(','),
+        :output => 'json',
+    }
+    options.merge!({:dimensions => @dimensions.join(',')}) if (@dimensions and @dimensions.size?)
+    options.merge!({:filters => @filters}) if @filters
+    options.merge!({:sort => @sort}) if @sort
+    options.merge!({:segment => @segment}) if @segment
+    options.merge!({:sampling_level => @sampling_level}) if @sampling_level
+    options.merge!({:include_empty_rows => @include_empty_rows}) if !@include_empty_rows.nil?
+    return options
+  end
+
+  def get_service
+    scope = 'https://www.googleapis.com/auth/analytics.readonly'
+    authorizer = Google::Auth::ServiceAccountCredentials.make_creds(
+        json_key_io: File.open(@key_file_path),
+        scope: scope
+    )
+
+    analytics = Google::Apis::AnalyticsV3::AnalyticsService.new
+    analytics.authorization = authorizer
+    return analytics
+  end
+
+  private
+
+  def is_num(a)
+    return (Float(a) and true) rescue false
+  end
+end # class LogStash::Inputs::GoogleAnalytics
+

data/logstash-input-google-analytics-daily.gemspec

@@ -0,0 +1,28 @@
+Gem::Specification.new do |s|
+  s.name = 'logstash-input-google-analytics-daily'
+  s.version = '0.1.0'
+  s.licenses = ['Apache License (2.0)']
+  s.summary = "Logstash plugin to pull daily reports from the Google Analytics v3 Core Reporting API."
+  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install logstash-input-google-analytics-daily."
+  s.authors = ["Shalvah"]
+  s.email = 'hello@shalvah.me'
+  s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 2.0.0", "< 3.0.0"
+  s.add_runtime_dependency 'stud', '>= 0.0.22'
+  s.add_runtime_dependency 'google-api-client', "0.37"
+  s.add_runtime_dependency 'googleauth'
+  s.add_development_dependency 'vcr'
+  s.add_development_dependency 'webmock'
+  s.add_development_dependency 'json'
+end

data/spec/inputs/google_analytics_daily_spec.rb

@@ -0,0 +1,47 @@
+# Tests currently nto in use
+#
+# # encoding: utf-8
+# require "logstash/inputs/google_analytics_daily"
+# require "vcr"
+# require "json"
+#
+# ENV['SSL_CERT_FILE'] = "/Users/rsavage/Downloads/cacert.pem"
+#
+# VCR.configure do |config|
+#   config.cassette_library_dir = File.join(File.dirname(__FILE__), '..', 'fixtures', 'vcr_cassettes')
+#   config.hook_into :webmock
+# end
+#
+# RSpec.describe LogStash::Inputs::GoogleAnalyticsDaily do
+#   describe "inputs/google_analytics_daily" do
+#     context "get users overview" do
+#       let(:options) do
+#         {
+#             "view_id" => "ga:213576060",
+#             "metrics" => ["ga:users","ga:sessions"],
+#             "key_file_path" => "C:/logstash-7.6.1/keyfile.json",
+#             "dates" => ['yesterday', '2020-04-05'],
+#             'dimensions' => ['ga:browser'],
+#             "interval" => nil,
+#
+#         }
+#       end
+#       let(:input) { LogStash::Inputs::GoogleAnalyticsDaily.new(options) }
+#       let(:expected_fields_result) { ["ga_pageviews"] }
+#       let(:queue) { [] }
+#       subject { input }
+#       it "loads pageviews" do
+#         #VCR.use_cassette("get_audience_overview") do
+#           subject.register
+#           subject.run(queue)
+#           expect(queue.length).to eq(1)
+#           e = queue.pop
+#           expected_fields_result.each do |f|
+#             expect(e.to_hash).to include(f)
+#           end
+#         #end
+#       end
+#     end
+#   end
+#
+# end

metadata

@@ -0,0 +1,160 @@
+--- !ruby/object:Gem::Specification
+name: logstash-input-google-analytics-daily
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Shalvah
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2020-04-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  name: logstash-core-plugin-api
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+  name: stud
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: '0.37'
+  name: google-api-client
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: '0.37'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: googleauth
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: vcr
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: webmock
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: json
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/plugin install logstash-input-google-analytics-daily.
+email: hello@shalvah.me
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- DEVELOPER.md
+- Gemfile
+- LICENSE
+- NOTICE.TXT
+- README.md
+- lib/logstash/inputs/google_analytics_daily.rb
+- logstash-input-google-analytics-daily.gemspec
+- spec/inputs/google_analytics_daily_spec.rb
+homepage: http://www.elastic.co/guide/en/logstash/current/index.html
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: input
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.6
+signing_key:
+specification_version: 4
+summary: Logstash plugin to pull daily reports from the Google Analytics v3 Core Reporting
+  API.
+test_files:
+- spec/inputs/google_analytics_daily_spec.rb