logstash-input-algolialogs 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4b7b99381d1d66c005b7cbdf39e0ea8c557228ef
+  data.tar.gz: 1d1cd13d1e3b75a00ed472d542c57eb92a5a3ee0
+SHA512:
+  metadata.gz: a55afe4032aadef7e04ce1d2956b3c3d4b15f8eedc73a53e7fec46f81a76e420697029a0bf1f1ee213b0d3722c376f6709a0b37235dc67a758476c1d8f9bb00a
+  data.tar.gz: 67ace32bce396881d437a629ddc6d784e160379a54c88589856ade51178578ffb5f978a9b4960bc04833690133204434d44d449f615fa723e1160036e8b6b3e4

data/DEVELOPER.md

@@ -0,0 +1,2 @@
+# logstash-input-example
+Example input plugin. This should help bootstrap your effort to write your own input plugin!

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,14 @@
+# Logstash Plugin
+
+This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
+
+It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
+
+## Documentation
+
+This plugin get logs from an algolia app (https://www.algolia.com/doc/rest_api#SectionLogs).
+
+Main idea is to retrieve logs from algolia and put it in elasticsearch (or elsewhere) after some filters.
+With Kibana or curiosity (https://github.com/pagesjaunes/curiosity).
+
+See an example of [conf](https://github.com/ErwanPigneul/logstash-input-algolialogs/blob/master/logstash-input-algolialogs.conf) 

data/Rakefile

@@ -0,0 +1 @@
+require "logstash/devutils/rake"

data/lib/logstash/inputs/algolialogs.rb

@@ -0,0 +1,50 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "stud/interval"
+
+# Generate a repeating message.
+#
+# This plugin is intented only as an example.
+
+class LogStash::Inputs::Algolialogs < LogStash::Inputs::Base
+  config_name "algolialogs"
+
+  # If undefined, Logstash will complain, even if codec is unused.
+  default :codec, "plain" 
+
+  # The application_id of algolia cluster.
+  config :application_id , :validate => :string, :required => true
+
+  # The api_key of algolia cluster (must be an admin key).
+  config :api_key , :validate => :string, :required => true
+
+  # The size of logs to retrieve.
+  config :length , :validate => :number, :default => 1000
+
+  # Set how frequently messages should be sent.
+  #
+  # The default, `1`, means send a message every second.
+  config :interval, :validate => :number, :default => 600
+
+  public
+  def register
+    require "algoliasearch"
+
+    Algolia.init :application_id => @application_id,
+                  :api_key        => @api_key
+
+  end # def register
+
+  def run(queue)
+    Stud.interval(@interval) do
+      logs = Algolia.get_logs(0, @length, true)
+      for log in logs["logs"]
+        event = LogStash::Event.new(log)
+        decorate(event)
+        queue << event
+      end
+    end # loop
+  end # def run
+
+end # class LogStash::Inputs::Example

data/logstash-input-algolialogs.conf

@@ -0,0 +1,56 @@
+input {
+  algolialogs {
+    application_id => "YOUR_APPID"
+    api_key => "YOUR_APIKEY"
+    length => 1000
+    interval => 60
+  }
+}
+
+filter {
+
+	# add geoip informations
+	geoip {
+		source => "ip"
+	}
+
+	# transform params to be searchable
+	ruby {
+		code => "require 'cgi'
+		if (event['query_params'])
+			event['query_params'] = CGI::parse(event['query_params'])
+		end"
+	}
+
+	# parse headers and keep just necessary
+	ruby {
+		code => "require 'net/http'
+		event['headers'] = {}
+        event['query_headers'].each_line { |line| 
+          params = line.split(': ')
+          if (params[0] == 'User-Agent' || params[0] == 'Referer' || params[0] == 'Origin')
+          	event['headers'][params[0].downcase] = params[1]
+          end
+        }"
+	}
+
+	# translate user agent
+	useragent {
+    	source => "[headers][user-agent]"
+    	target => "[headers][useragent]"
+	}
+
+	# remove unusual fields and convert some string to integer
+	mutate {
+		remove_field => [ "answer", "url", "ip", "[headers][user-agent]", "query_body", "query_headers"]
+		convert => { "nb_api_calls" => "integer" }
+		convert => { "processing_time_ms" => "integer" }
+		convert => { "query_nb_hits" => "integer" }
+	}
+}
+
+output {
+  stdout {
+    codec => rubydebug
+  }
+}

data/logstash-input-algolialogs.gemspec

@@ -0,0 +1,29 @@
+Gem::Specification.new do |s|
+  s.name = 'logstash-input-algolialogs'
+  s.version         = '0.1.0'
+  s.licenses = ['Apache License (2.0)']
+  s.summary = "Retrieve logs from algolia."
+  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors = ["epigneul"]
+  s.email = 'erwan.pigneul@gmail.com'
+  s.homepage = "https://github.com/ErwanPigneul/logstash-input-algolialogs"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = `git ls-files`.split($\)
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'
+  s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'stud'
+  s.add_runtime_dependency 'algoliasearch'
+
+  s.add_development_dependency 'logstash-devutils'
+
+
+end

data/spec/inputs/algolialogs_spec.rb

@@ -0,0 +1 @@
+require "logstash/devutils/rspec/spec_helper"

metadata

@@ -0,0 +1,131 @@
+--- !ruby/object:Gem::Specification
+name: logstash-input-algolialogs
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- epigneul
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-06-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash-core
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: stud
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: algoliasearch
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email: erwan.pigneul@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- DEVELOPER.md
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/logstash/inputs/algolialogs.rb
+- logstash-input-algolialogs.conf
+- logstash-input-algolialogs.gemspec
+- spec/inputs/algolialogs_spec.rb
+homepage: https://github.com/ErwanPigneul/logstash-input-algolialogs
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: input
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.5
+signing_key:
+specification_version: 4
+summary: Retrieve logs from algolia.
+test_files:
+- spec/inputs/algolialogs_spec.rb