logstash-filter-mautic 0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4989fb37b968f02e249a43e4388a620314a98d3b
+  data.tar.gz: 81ba8474b2407c61c04effb51ebe7779c9b725f5
+SHA512:
+  metadata.gz: 82ae3685b774aeb7e4719136aed14ffa4522dde97a8c34ac4ca93dbbc867ba4edb9b759ed92b017618287d7c49b2ff9e7628b0d101399ebddefd4600d52d3aeb
+  data.tar.gz: 7e4d5beea13e819afada5b0624c060285791c97ae6a5e1bba2ea25d6694ebdda58bb4feff4453bda1527cce6b0ed29d1ea05a84902e1f769b2802bcb53c9d888

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## 2.0.0
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
+   instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
+ - Dependency on logstash-core update to 2.0
+

data/CONTRIBUTORS

@@ -0,0 +1,11 @@
+The following is a list of people who have contributed ideas, code, bug
+reports, or in general have helped logstash along its way.
+
+Contributors:
+* Aaron Mildenstein (untergeek)
+* Pier-Hugues Pellerin (ph)
+
+Note: If you've sent us patches, bug reports, or otherwise contributed to
+Logstash, and you aren't on the list above and want to be, please let us know
+and we'll make sure you're here. Contributions from folks like you are what make
+open source awesome.

data/DEVELOPER.md

@@ -0,0 +1,2 @@
+# logstash-filter-mautic
+Example filter plugin. This should help bootstrap your effort to write your own filter plugin!

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+ruby '1.9.3', :engine => 'jruby', :engine_version => '1.7.22'
+
+gem "logstash-core", :github => "elastic/logstash", :branch => "2.0"
+gem "gem"
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/NOTICE.TXT

@@ -0,0 +1,5 @@
+Elasticsearch
+Copyright 2012-2015 Elasticsearch
+
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).

data/README.md

@@ -0,0 +1,92 @@
+# Logstash Plugin
+
+[![Build
+Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-example-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-example-unit/)
+
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
+
+It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
+
+
+1. Install this plugin using 
+```
+./plugin install logstash-filter-mautic
+```
+
+
+2. Setup your Logstash configuration like so
+
+```
+input {
+	http {
+		host => "127.0.0.1"
+		port => "5543"
+		type => "mautic-lead"
+	}
+
+}
+filter {
+
+if [type] == "mautic-lead" {
+		mautic {
+			source => "message"
+			remove_field => ["headers", "message", "host"]
+		}
+		if [type] == "lead"{
+			mutate {
+				replace => { "@timestamp" => "%{dateAdded}"}
+			}
+		}
+		if [type] == "form_submission"{
+			mutate {
+				replace => { "@timestamp" => "%{dateSubmmited}"}
+			}
+		}
+		if [type] == "email"{
+			mutate {
+				replace => { "@timestamp" => "%{dateSent}"}
+			}
+		}
+		if [type] == "page_hit"{
+			mutate {
+				replace => { "@timestamp" => "%{dateHit}"}
+			}
+		}
+		
+	}
+
+}
+output {
+	if [type] == "lead" {
+		elasticsearch { 
+			hosts => ["localhost:9200"] 
+			index => ["mautic-leads"]
+			#document_id => "%{[leadid]}"
+		}
+	}
+	else if [type] == "form_submission" {
+		elasticsearch { 
+			hosts => ["localhost:9200"] 
+			index => ["mautic-leads"]
+			document_id => "F%{[leadid]}-%{[form][id]}-%{[submissionid]}"
+			routing => "%{[leadid]}"
+		}
+	}
+	else if [type] == "page_hit" {
+		elasticsearch { 
+			hosts => ["localhost:9200"] 
+			index => ["mautic-leads"]
+			routing => "%{[leadid]}"
+		}
+	}
+	else if [type] == "email" {
+		elasticsearch { 
+			hosts => ["localhost:9200"] 
+			index => ["mautic-leads"]
+			document_id => "E%{[leadid]}-%{[email][id]}-%{[emailopenid]}"
+			routing => "%{[leadid]}"
+		}
+	}
+	}
+
+```

data/lib/logstash/filters/mautic.rb

@@ -0,0 +1,325 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+require "logstash/json"
+require "logstash/timestamp"
+# This example filter will replace the contents of the default 
+# message field with whatever you specify in the configuration.
+#
+# It is only intended to be used as an example.
+class LogStash::Filters::Mautic < LogStash::Filters::Base
+
+  # Setting the config_name here is required. This is how you
+  # configure this filter from your Logstash config.
+  #
+  # filter {
+  #   example {
+  #     message => "My message..."
+  #   }
+  # }
+  #
+  config_name "mautic"
+  
+  # Replace the message with this value.
+  config :source, :validate => :string, :required => true
+  config :tag_on_failure, :validate => :array, :default => ["_mauticparsefailure"]
+  
+
+  public
+  def register
+    # Add instance variables 
+  end # def register
+
+  public
+  def filter(event)
+
+    begin
+      json_data = LogStash::Json.load(event[@source])
+    rescue => e
+       tag = "_jsonparsefailure"
+       event["tags"] ||= []
+       event["tags"] << tag unless event["tags"].include?(tag)
+       @logger.warn("Trouble parsing json", :source => @source,
+                    :raw => event[@source], :exception => e)
+       @logger.warn("Trouble parsing json", :exception => e)
+       return
+    end
+    tag = "_mauticparsefailure"
+    if json_data
+      # Replace the event message with our message as configured in the
+      # config file.
+      if json_data.is_a?(Hash)
+        matches = json_data.select{|k,v| k =~ /mautic.lead_post_save_update|mautic.lead_post_save_new|mautic.lead_points_change/}
+      end
+      if matches 
+        matches.each do |k,v|
+          if v.is_a?(Hash)                    ##############Go here if it is a JSON object
+            
+            parsed_data = processNewUpdateLead(v,k)
+
+            event_cloned = event.clone
+            event_cloned.timestamp = parsed_data['dateAdded']
+
+            parsed_data.each do |k1,v1|       ## Pull in the data
+              event_cloned[k1] = v1
+            end
+
+            filter_matched(event_cloned)
+            yield event_cloned
+          else
+            v.each do |k2,v2|             ######################## Go here if it is JSON array
+           
+              parsed_data = processNewUpdateLead(k2,k)
+
+              event_cloned = event.clone
+              event_cloned.timestamp = parsed_data['dateAdded']
+
+
+              parsed_data.each do |k1,v1|       ## Pull in the data
+                event_cloned[k1] = v1
+              end
+
+              filter_matched(event_cloned)
+              yield event_cloned
+            end # end loop
+          end
+        end
+      end  # end if
+
+      if json_data['mautic.email_on_open']
+        if json_data['mautic.email_on_open'].is_a?(Hash)
+          parsed_data = processEmail(json_data['mautic.email_on_open'])
+          
+          event_cloned = event.clone
+          event_cloned.timestamp = parsed_data['dateSent']
+          
+          parsed_data.each do |k1,v1|       ## Pull in the data
+            event_cloned[k1] = v1
+          end
+          
+          filter_matched(event_cloned)
+          yield event_cloned
+        
+        else
+          json_data['mautic.email_on_open'].each do |value|
+            parsed_data = processEmail(value)
+
+            event_cloned = event.clone
+            event_cloned.timestamp = parsed_data['dateSent']
+            
+            parsed_data.each do |k1,v1|       ## Pull in the data
+              event_cloned[k1] = v1
+            end
+            
+            filter_matched(event_cloned)
+            yield event_cloned
+          
+          end # end loop
+        end
+      end  # end if
+
+      if json_data['mautic.form_on_submit']
+
+        if json_data['mautic.form_on_submit'].is_a?(Array)
+          json_data['mautic.form_on_submit'].each do |value|
+            parsed_data = processForm(value)
+
+            event_cloned = event.clone
+            event_cloned.timestamp = parsed_data['dateSubmitted']
+
+            parsed_data.each do |k1,v1|       ## Pull in the data
+              event_cloned[k1] = v1
+            end
+
+            filter_matched(event_cloned)
+            yield event_cloned
+          end # end loop
+        else
+          parsed_data = processForm(json_data['mautic.form_on_submit'])
+
+          event_cloned = event.clone
+          event_cloned.timestamp = parsed_data['dateSubmitted']
+
+          
+          parsed_data.each do |k1,v1|       ## Pull in the data
+            event_cloned[k1] = v1
+          end
+
+          filter_matched(event_cloned)
+          yield event_cloned
+        end  # end if
+      end  # end if
+
+      if json_data['mautic.page_on_hit']
+        if json_data['mautic.page_on_hit'].is_a?(Array)
+          json_data['mautic.page_on_hit'].each do |value|
+            parsed_data = processHit(value)
+
+            event_cloned = event.clone
+            event_cloned.timestamp = parsed_data['dateHit']
+
+            parsed_data.each do |k1,v1|       ## Pull in the data
+              event_cloned[k1] = v1
+            end
+
+            filter_matched(event_cloned)
+            yield event_cloned
+          end # end loop
+        else
+          parsed_data = processHit(json_data['mautic.page_on_hit'])
+
+          event_cloned = event.clone
+          event_cloned.timestamp = parsed_data['dateHit']
+          
+          parsed_data.each do |k1,v1|       ## Pull in the data
+            event_cloned[k1] = v1
+          end
+          filter_matched(event_cloned)
+          yield event_cloned
+        end  # end if
+      end
+
+    end # end if
+
+    # filter_matched should go in the last line of our successful code
+    event.cancel   #As we have cloned the event we need to cancel the original one
+  end # def filter
+
+  def processHit(json_data)
+    parsed_data = json_data['hit']
+    parsed_data['leadid'] = parsed_data['lead']['id'].to_i
+    parsed_data.delete('lead')
+
+
+    parsed_data['parent'] = parsed_data['leadid']
+    parsed_data['type'] = "page_hit"
+
+    if parsed_data['dateHit']
+      parsed_data['dateHit'] = LogStash::Timestamp.parse_iso8601(parsed_data['dateHit'])
+    end
+
+    if parsed_data['dateLeft']
+      parsed_data['dateLeft'] = LogStash::Timestamp.parse_iso8601(parsed_data['dateLeft'])
+    end
+    return parsed_data
+  end
+
+  def processForm(json_data)
+    parsed_data = json_data['submission']
+    parsed_data['leadid'] = parsed_data['lead']['id'].to_i
+    parsed_data['submissionid'] = parsed_data['id']
+
+
+    parsed_data.delete('id')
+    parsed_data.delete('lead')
+    parsed_data.delete('ipAddress')
+
+    if parsed_data['dateSubmitted']
+      parsed_data['dateSubmitted'] = LogStash::Timestamp.parse_iso8601(parsed_data['dateSubmitted'])
+    end
+
+    parsed_data['type'] = "form_submission"
+    parsed_data['parent'] = parsed_data['leadid']
+
+
+    
+    # num =0
+    # parsed_data["ipAddresses"] = []
+    # parsed_data['ipAddresses'].each do |k2,v2|
+    #   formData['ipAddresses'][num] = k2
+    #   num +=1
+    # end  
+    return parsed_data
+  end
+
+  def processNewUpdateLead(json_data,key)
+    parsed_data = newLeadFilter(json_data)
+    parsed_data['leadid'] = parsed_data['id'].to_i
+    parsed_data.delete('id')
+
+    if parsed_data['dateIdentified'] != nil
+      parsed_data['dateIdentified'] = LogStash::Timestamp.parse_iso8601(parsed_data['dateIdentified'])
+    end
+
+    if parsed_data['dateAdded'] != nil
+      parsed_data['dateAdded'] = LogStash::Timestamp.parse_iso8601(parsed_data['dateAdded'])
+    end
+
+    if parsed_data['dateModified'] != nil
+      parsed_data['dateModified'] = LogStash::Timestamp.parse_iso8601(parsed_data['dateModified'])
+    end
+
+      parsed_data['type'] = "lead"
+    return parsed_data
+  end
+
+  def processEmail(json_data)
+    parsed_data = json_data['stat']
+    parsed_data['leadid'] = parsed_data['lead']['id'].to_i
+    parsed_data['emailopenid'] = parsed_data['id']
+
+    parsed_data.delete("lead")
+    parsed_data.delete("id")
+
+    parsed_data['type'] = "email"
+    parsed_data['parent'] = parsed_data['leadid']
+
+
+    if parsed_data['dateSent'] != nil
+      parsed_data['dateSent'] = LogStash::Timestamp.parse_iso8601(parsed_data['dateSent'])
+    end    
+
+    if parsed_data['dateRead'] != nil
+      parsed_data['dateRead'] = LogStash::Timestamp.parse_iso8601(parsed_data['dateRead'])
+    end
+    if parsed_data['lastOpened'] != nil
+      parsed_data['lastOpened'] = LogStash::Timestamp.parse_iso8601(parsed_data['lastOpened'])
+    end
+    
+    return parsed_data
+  end
+
+  def newLeadFilter(json_data)
+
+    formData = {}
+    json_data["lead"].each do |k1,v1|
+      if k1 === "ipAddresses"
+        num = 0
+        formData["ipAddresses"] = []
+        v1.each do |k2,v2|
+          formData[k1][num] = k2
+          num +=1
+        end  
+      elsif k1 === "tags"
+        num =0
+        formData["mautic_tags"] = []
+        v1.each do |v2|
+          formData["mautic_tags"][num] =v2
+          num +=1
+        end
+      elsif v1.is_a?(Hash)
+        v1.each do |k2,v2|
+          if v2.is_a?(Hash)
+            v2.each do |k3,v3|
+              if v3.is_a?(Hash)
+                v3.each do |k4,v4| 
+                  if v4 != nil and k4 === 'value'
+                    formData[k3] = v4
+                  end
+                end
+              elsif v3 != nil
+                formData[k3] = v3
+              end
+            end
+          elsif v2 != nil and !(k2 === 'fields') and !(k1 === 'owner')
+            formData[k2] = v2
+          end
+        end
+      elsif v1 != nil
+        formData[k1] = v1
+      end
+    end
+    return formData
+  
+  end
+end # class LogStash::Filters::Mautic