logstash-filter-sumo 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9791dfa89ad6be36898f04ab8f49d05ad9bbb44b
+  data.tar.gz: 310bcb75636ebae5a57bc1e5d4d8ee9704e739cb
+SHA512:
+  metadata.gz: d12ee98e71095afe03c0ae1b090917081459c7c76c8d512abb8c944949395b725c97c33808bcd0ec04b790051ac5ff86e44452f2e37083d2a298f5f2bce8aa36
+  data.tar.gz: b3946cda9ea2c5526aefda2fdd527c731ebb43584054a189b10bdb402f003f152aebdbb50f697cdb16a230fd01a48ddd0463c98d9862183ffab3d658166c2586

data/CHANGELOG.md

@@ -0,0 +1,4 @@
+## 1.0.0
+ - Plugin for parsing access log pages from the tv2 sumo system
+ 
+

data/CONTRIBUTORS

@@ -0,0 +1,6 @@
+Version 1.0.0
+
+- Leif Terje Fonnes
+
+
+

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+gemspec
+
+group :development do
+  gem 'guard',"1.8.3"
+  gem 'guard-jruby-rspec', git: 'https://github.com/jkutner/guard-jruby-rspec.git'
+end

data/LICENSE

@@ -0,0 +1 @@
+Noe lisens informasjon

data/README.md

@@ -0,0 +1,78 @@
+# Logstash Plugin
+
+[![Build
+Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-example-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-example-unit/)
+
+This is a plugin for [Logstash](https://github.com/elastic/logstash).
+
+## Documentation
+
+## Developing
+
+### 1. Plugin Developement and Testing
+
+#### Code
+- To get started, you'll need JRuby with the Bundler gem installed.
+
+- Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization. We also provide [example plugins](https://github.com/logstash-plugins?query=example).
+
+- Install dependencies
+```sh
+bundle install
+```
+
+#### Test
+
+- Update your dependencies
+
+```sh
+bundle install
+```
+
+- Run tests
+
+```sh
+bundle exec rspec
+```
+
+### 2. Running your unpublished Plugin in Logstash
+
+#### 2.1 Run in a local Logstash clone
+
+- Edit Logstash `Gemfile` and add the local plugin path, for example:
+```ruby
+gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
+```
+- Install plugin
+```sh
+bin/plugin install --no-verify
+```
+- Run Logstash with your plugin
+```sh
+bin/logstash -e 'filter {awesome {}}'
+```
+At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
+
+#### 2.2 Run in an installed Logstash
+
+You can use the same **2.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using:
+
+- Build your plugin gem
+```sh
+gem build logstash-filter-awesome.gemspec
+```
+- Install the plugin from the Logstash home
+```sh
+bin/plugin install /your/local/plugin/logstash-filter-awesome.gem
+```
+- Start Logstash and proceed to test the plugin
+
+## Contributing
+
+All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
+
+Programming is not a required skill. Whatever you've seen about open source and maintainers or community members  saying "send patches or die" - you will not see that here.
+
+It is more important to the community that you are able to contribute.
+
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.

data/lib/logstash/filters/sumo.rb

@@ -0,0 +1,109 @@
+# encoding: utf-8
+require "logstash/filters/base"
+require "logstash/namespace"
+require 'uri'
+
+
+class LogStash::Filters::Sumo < LogStash::Filters::Base
+  config_name "sumo"
+
+  config :page, :validate => :string, :default => ""
+
+
+  def wzlive_parse(uarray)
+    web_stream = uarray[3].split(":")[1]
+    video_type = uarray.last.split(".").last
+    asset_id = uarray[4].split(".").first.split("_").first.to_i
+
+    {path_type: "wzlive", dist_type: "live", web_stream: web_stream, video_type: video_type, asset_id: asset_id}
+  end
+
+  def dashvod_parse(uarray)
+    asset_id = uarray[3].split(":")[1].split("_").first.to_i
+    last_seg = uarray.last.split(".")
+    video_type = last_seg.last
+    chunk_type = last_seg.first.include?('chunk_ctaudio') ? :audio : (last_seg.first.include?('chunk_ctvideo') ? :video : :other)
+
+    {path_type: "dashvod", dist_type: "vod", video_type: video_type, asset_id: asset_id, chunk_type: chunk_type}
+  end
+
+  def vod2_parse uarray
+    asset_id = uarray[3].split(":")[1].split("_").first.to_i
+    last_seg = uarray.last.split(".")
+    video_type = last_seg.last
+
+#,video_type: video_type, asset_id: asset_id,chunk_type: chunk_type
+    {path_type: "vod2", dist_type: "vod", video_type: video_type, asset_id: asset_id}
+  end
+
+  # /ismusp/2013-10-22/N_RYANAIR_221013_TOB_NET(763009_R22MP41000).mp4
+  def ismusp_parse uarray
+    last_seg = uarray.last.split(".")
+    asset_id = last_seg[0].split("(")[1].split("_").first.to_i
+    video_type = last_seg.last
+
+    {path_type: "ismusp", dist_type: "vod", video_type: video_type, asset_id: asset_id}
+  end
+
+
+  # /ismusp/2013-10-22/N_RYANAIR_221013_TOB_NET(763009_R22MP41000).mp4
+  def ismvod_parse uarray
+    last_seg = uarray.last.split(".")
+    asset_id = uarray[3].split(":")[1].split(".").first.to_i
+    video_type = last_seg.last
+
+    {path_type: "ismvod", dist_type: "vod", video_type: video_type, asset_id: asset_id}
+  end
+
+  # /coraid_ism_1/2015-06-11/Bloggerne_4_(924507_ISM).ism/Manifest
+  # /netapp_ism_10/2015-04-01/R_BLOGGERNE4_EP25_G02371(902135_R283ISMH).ism/Manifest
+  def ism_parse uarray
+    page_type = uarray.pop.split(".")
+    page_info = uarray.pop
+    asset_id = page_info.split("(")[1].split("_")[0].to_i
+
+    {path_type: "ism", dist_type: "vod", video_type: page_type.last, asset_id: asset_id}
+  end
+
+  def parse_page page
+    return {} unless page.to_s.strip.length > 0
+    uri = URI.parse(page)
+    uarray = uri.path.split("/")
+    page_type = uarray[-2]
+
+    if uarray[1] == 'wzlive'
+      wzlive_parse(uarray)
+    elsif uarray[1] == 'dashvod'
+      dashvod_parse(uarray)
+    elsif uarray[1] == 'vod2'
+      vod2_parse(uarray)
+    elsif uarray[1] == 'ismusp'
+      ismusp_parse(uarray)
+    elsif uarray[1] == 'ismvod'
+      ismvod_parse(uarray)
+    elsif page_type.to_s.include?(".ism")
+      ism_parse(uarray)
+    else
+      {}
+    end
+  end
+
+
+  public
+  def register
+    # Add instance variables 
+  end
+
+  # def register
+
+  public
+  def filter(event)
+    if @page
+      data = parse_page(@page)
+      data.each { |k, v|event[k.to_s]=v }
+    end
+
+    # filter_matched should go in the last line of our successful code
+    filter_matched(event)
+  end # def filter
+end # class LogStash::Filters::Sumo

data/logstash-filter-sumo.gemspec

@@ -0,0 +1,26 @@
+Gem::Specification.new do |s|
+  s.name = 'logstash-filter-sumo'
+  s.version         = '1.0.0'
+  s.licenses = ['Apache License (2.0)']
+  s.summary = "This filter extracts interessting SUMO values from the page element if present."
+  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors = ["TV 2","Leif Terje Fonnes"]
+  s.email = 'leffen@gmail.com'
+  s.homepage = "http://tv2.no"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE']
+
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 3.0.0"
+  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'rspec', '~> 2.7'
+
+end

data/spec/filters/sumo_spec.rb

@@ -0,0 +1,112 @@
+# encoding: utf-8
+require 'spec_helper'
+require "logstash/filters/sumo"
+
+describe LogStash::Filters::Sumo do
+  describe "Dashvod" do
+    let(:config) do <<-CONFIG
+      filter {
+        sumo {
+          page => "/dashvod/_definst_/amlst:993514_ps20000_pd1291830.smil/chunk_ctaudio_cfm4s_ridp0a1r1_cs28929600_mpd.m4s"
+        }
+      }
+    CONFIG
+    end
+
+    sample("page"=>"" ) do
+      expect(subject).to include("path_type")
+      expect(subject['path_type']).to eq('dashvod')
+      expect(subject['dist_type']).to eq('vod')
+      expect(subject['video_type']).to eq('m4s')
+      expect(subject['asset_id']).to eq(993514)
+    end
+
+  end
+
+  describe "ismusp" do
+    let(:config) do <<-CONFIG
+      filter {
+        sumo {
+          page => "/ismusp/2013-10-22/N_RYANAIR_221013_TOB_NET(763009_R22MP41000).mp4"
+        }
+      }
+    CONFIG
+    end
+
+    sample("page"=>"" ) do
+      expect(subject).to include("path_type")
+      expect(subject['path_type']).to eq('ismusp')
+      expect(subject['dist_type']).to eq('vod')
+      expect(subject['video_type']).to eq('mp4')
+      expect(subject['asset_id']).to eq(763009)
+    end
+
+  end
+
+  describe "vod2" do
+    let(:config) do <<-CONFIG
+      filter {
+        sumo {
+          page => "/vod2/_definst_/amlst:1027448_ps55210_pd1373700_HRDNGR.smil/media_b6378000_43.ts"
+        }
+      }
+    CONFIG
+    end
+
+    sample("page"=>"" ) do
+      expect(subject).to include("path_type")
+      expect(subject['path_type']).to eq('vod2')
+      expect(subject['dist_type']).to eq('vod')
+      expect(subject['video_type']).to eq('ts')
+      expect(subject['asset_id']).to eq(1027448)
+    end
+
+  end
+
+  describe "wzlive" do
+    let(:config) do <<-CONFIG
+      filter {
+        sumo {
+          page => "/wzlive/_definst_/amlst:LS6/433127.smil/manifest.mpd"
+        }
+      }
+    CONFIG
+    end
+
+    sample("page"=>"" ) do
+      expect(subject).to include("path_type")
+      expect(subject['path_type']).to eq('wzlive')
+      expect(subject['dist_type']).to eq('live')
+      expect(subject['video_type']).to eq('mpd')
+      expect(subject['asset_id']).to eq(433127)
+    end
+
+  end
+
+
+  describe "wzlive" do
+    let(:config) do <<-CONFIG
+      filter {
+        sumo {
+          page => "/ismvod/_definst_/amlst:1027479.smil.ism/QualityLevels(126793)/Fragments(audio=143034920).isma"
+        }
+      }
+    CONFIG
+    end
+
+    sample("page"=>"" ) do
+      expect(subject).to include("path_type")
+      expect(subject['path_type']).to eq('ismvod')
+      expect(subject['dist_type']).to eq('vod')
+      expect(subject['video_type']).to eq('isma')
+      expect(subject['asset_id']).to eq(1027479)
+    end
+
+  end
+
+
+
+
+
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,2 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification
+name: logstash-filter-sumo
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- TV 2
+- Leif Terje Fonnes
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2016-03-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+  name: rspec
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email: leffen@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/filters/sumo.rb
+- logstash-filter-sumo.gemspec
+- spec/filters/sumo_spec.rb
+- spec/spec_helper.rb
+homepage: http://tv2.no
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: filter
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
+specification_version: 4
+summary: This filter extracts interessting SUMO values from the page element if present.
+test_files:
+- spec/filters/sumo_spec.rb
+- spec/spec_helper.rb