logstash-filter-sumo 3.0.22 → 3.0.27

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2219d28e3c848821f52a9eb67ea246058b956a63
-  data.tar.gz: 0b9039e60627e1136669352b0d565638eb6ecd03
+  metadata.gz: f33a38e9d825ac9c11e56c73f870357cacf217b5
+  data.tar.gz: 2e8cf5b1fe19a4dae2b0b182bb9758ff3ea30c62
 SHA512:
-  metadata.gz: bbc8e82d378cf866ebceab4878ea5c327969d9efb18fa4925bdce30914dd58fb9bc832162ebbccf8a2c274b09ec24639b1827138ce1c19feef50105e883df526
-  data.tar.gz: b7eb743fe9d991cae62f18679a00acf8da007d33917f983127df1a8eb69e9ac82859a9053c6204bd7932985376f63573d380ac57b6b1de7714fe86a2f49ebc19
+  metadata.gz: a83ac97a4e74948f3953b66ad26f78f9288627856dcee9f0781d4a96a913678e0c9e7d4ffc226de9a3c02403a483b3abf822f92bc2f43f05f201b343907bf44a
+  data.tar.gz: cfa280d10677fb1cbd2d0e8c58b803156bc12bd2bef4b0d84306d49602e4617b38654803f688d2557a6b842a1eb276de709d650fd059afe5d51a038c9f3b4f51

data/README.md

@@ -14,7 +14,10 @@ This projects needs jruby, so use "rvm use jruby". Tested and deployed with jrub
 3.0.20 Added support for hdsvod
 3.0.21 Added support for vod4
 3.0.22 Added support for /origin1-live.akamai.tv2.no/live/727992.isml/Manifest
-
+3.0.23 Fix of bug in parsing origin1-live urls
+3.0.25 Added support for akamai cache_key
+3.0.26 Added support for /origin2-od.akamai.tv2.no/ismusp/isi_mp4_0/2014-04-25/G018902sumo(809735_ISMUSP).ism/Manifest
+3.0.27 Added exception handling for page_parse part
 
 ## Build
 Update the version numder in logstash-filter-sumo.gemspec

data/lib/logstash/filters/sumo.rb

@@ -52,6 +52,16 @@ class LogStash::Filters::Sumo < LogStash::Filters::Base
   end
 
   def ismusp_parse uarray
+    if uarray.last == "Manifest"
+      # handle  /origin2-od.akamai.tv2.no/ismusp/isi_mp4_0/2014-04-25/G018902sumo(809735_ISMUSP).ism/Manifest
+      last_seg = uarray[-2].split(".")
+      asset_id = last_seg[0].split("(")[1].split("_").first.to_i
+      video_type = last_seg.last
+
+      return {path_type: "ismusp", dist_type: "vod", video_type: video_type, asset_id: asset_id,vtype: "manifest"}
+
+    end
+
     last_seg = uarray.last.split(".")
     asset_id = last_seg[0].split("(")[1].split("_").first.to_i
     video_type = last_seg.last
@@ -90,7 +100,13 @@ class LogStash::Filters::Sumo < LogStash::Filters::Base
   def ism_parse uarray
     page_type = uarray.pop.split(".")
     page_info = uarray.pop
-    asset_id = page_info.split("(")[1].split("_")[0].to_i
+
+    ua = page_info.split("(")
+    puts "ism_parse ua = #{ua}"
+    asset_id = nil
+    if ua.count > 1
+      asset_id = ua[1].split("_")[0].to_i
+    end
 
     {path_type: "ism", dist_type: "vod", video_type: page_type.last, asset_id: asset_id}
   end
@@ -111,16 +127,14 @@ class LogStash::Filters::Sumo < LogStash::Filters::Base
   def isml_parse uarray
     # ["", "live", "727992.isml"]
 
-    # /origin1-live.akamai.tv2.no/live/727992.isml/Manifest
-    if uarray.last == "Manifest" && uarray[2]== "live"
-      aparts = uarray[3].split(".")
+    # /live/727992.isml/Manifest
+    if uarray.length > 2 && uarray.last == "Manifest" && uarray[1]== "live"
+      aparts = uarray[2].to_s.split(".")
       asset_id = aparts[0]
 
       return {path_type: "live", dist_type: "live", video_type: "ism", vtype: "manifest", asset_id: asset_id.to_s.to_i}
     end
 
-
-
     page_name = uarray.pop
     splitted_page = page_name.split(".")
 
@@ -198,45 +212,66 @@ class LogStash::Filters::Sumo < LogStash::Filters::Base
 
   def parse_page page
     return {} unless page.to_s.strip.length > 0
-    uri = URI.parse(page)
-    uarray = uri.path.split("/")
-    return {} if uarray.length < 2
-
-    page_type = uarray[-2]
-
-    return {} unless page_type && uarray.count > 0
-
-    root_page = uarray[1].to_s.downcase
-
-    #puts "VOD Startswith #{root_page} #{root_page.start_with? 'vod'}"
-
-    if root_page == 'wzlive'
-      wzlive_parse(uarray)
-    elsif root_page.include?('dashvod')
-      dashvod_parse(uarray)
-    elsif root_page == 'vod2'
-      vod2_parse(uarray)
-    elsif root_page.start_with? 'vod'
-      vod_parse(uarray)
-    elsif root_page == 'ismusp'
-      ismusp_parse(uarray)
-    elsif root_page.include?('ismvod')
-      ismvod_parse(uarray)
-    elsif root_page.include?('hdsvod')
-      hdsvod_parse(uarray)
-    elsif root_page == 'out'
-      out_parse uarray
-    elsif page_type.to_s.include?(".isml") && root_page.to_s.include?("live")
-      isml_parse(uarray)
-    elsif page_type.to_s.include?(".ism")
-      ism_parse(uarray)
-    elsif root_page == 'live'
-      isml_parse uarray
-    elsif root_page.include?("healthcheck")
-      {path_type: "healthcheck", dist_type: "healthcheck", video_type:"healthcheck"}
-    else
-      {}
+    begin
+      rc = {page: page}
+      uri = URI.parse(page)
+
+     # puts "URI = #{uri} page=#{page}"
+
+      uarray = uri.path.split("/")
+      return {} if uarray.length < 2
+
+      cache_key = ""
+      start_word = uarray[1].to_s
+      if start_word.include?("origin") && start_word.include?("akamai")
+        cache_key = uarray.delete_at(1)
+   #     puts "CACHEKEY #{cache_key}"
+      end
+
+
+      page_type = uarray[-2]
+
+      return {} unless page_type && uarray.count > 0
+
+      root_page = uarray[1].to_s.downcase
+
+      #puts "root_page #{root_page} page_type=#{page_type} page=#{page}"
+
+      rc = {}
+
+      if root_page == 'wzlive'
+        rc = wzlive_parse(uarray)
+      elsif root_page.include?('dashvod')
+        rc = dashvod_parse(uarray)
+      elsif root_page == 'vod2'
+        rc = vod2_parse(uarray)
+      elsif root_page.start_with? 'vod'
+        rc = vod_parse(uarray)
+      elsif root_page == 'ismusp'
+        rc = ismusp_parse(uarray)
+      elsif root_page.include?('ismvod')
+        rc = ismvod_parse(uarray)
+      elsif root_page.include?('hdsvod')
+        rc = hdsvod_parse(uarray)
+      elsif root_page == 'out'
+        rc = out_parse uarray
+      elsif page_type.to_s.include?(".isml") && root_page.to_s.include?("live")
+        rc = isml_parse(uarray)
+      elsif page_type.to_s.include?(".ism")
+        rc = ism_parse(uarray)
+      elsif root_page == 'live'
+        rc = isml_parse uarray
+      elsif root_page.include?("healthcheck")
+        rc = {path_type: "healthcheck", dist_type: "healthcheck", video_type:"healthcheck"}
+      end
+      rc[:cache_key]= cache_key if cache_key.length > 0
+    rescue => e
+      rc[:exception]= e.to_json
+      rc[:page] = page
+      puts "SUMO parse_page #{e} rc=#{rc}"
+      @logger.error("SUMO.parse_page> ", :page => page, :exception => e)
     end
+    rc
   end
 
 
@@ -250,25 +285,33 @@ class LogStash::Filters::Sumo < LogStash::Filters::Base
   public
   def filter(event)
 
-    return unless filter?(@field)
-
-    if @field
-      fieldValue = event.get(@field)
-      
-      if fieldValue
-        begin
-          data = parse_page(fieldValue)
-          data.each { |k, v| event.set(k.to_s,v) }
-         # puts ("SUMO> #{fieldValue}  Event #{event} DATA: #{data}")
-        rescue => e
-          @logger.error("SUMO> ", :pg => fieldValue, :exception => e)
+    begin
+      return unless filter?(@field)
+
+      if @field
+        fieldValue = event.get(@field)
+
+        if fieldValue
+          begin
+            data = parse_page(fieldValue)
+            data.each { |k, v| event.set(k.to_s,v) } if data
+          rescue => e
+            sep = "\n"
+            puts "Error : #{e} data = #{data} backtrace=#{e.backtrace[0..2].join(sep)}"
+            @logger.error("SUMO> ", :pg => fieldValue, :exception => e)
+          end
         end
       end
-    end
 
 
-    # filter_matched should go in the last line of our successful code
-    filter_matched(event)
+      # filter_matched should go in the last line of our successful code
+      rc = filter_matched(event)
+
+    rescue => e
+      puts "SUMO error #{e} #fieldvalue #{fieldValue} "
+      @logger.error("SUMO> ", :pg => fieldValue, :exception => e)
+    end
+    rc
   end # def filter
 
 

data/logstash-filter-sumo.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-filter-sumo'
-  s.version         = '3.0.22'
+  s.version         = '3.0.27'
   s.licenses = ['Apache License (2.0)']
   s.summary = "This filter extracts interessting SUMO values from the page element if present."
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -20,8 +20,6 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
-  s.add_runtime_dependency 'redis'
-  s.add_development_dependency 'rspec', '~> 3.1'
   s.add_development_dependency 'logstash-devutils'
-
+  s.add_development_dependency 'rspec', '~> 3.1'
 end

data/spec/filters/sumo_spec.rb

@@ -39,6 +39,17 @@ describe LogStash::Filters::Sumo do
       expect(subject.get('asset_id')).to eq(763009)
     end
 
+    #
+    sample("page"=>"/origin2-od.akamai.tv2.no/ismusp/isi_mp4_0/2014-04-25/G018902sumo(809735_ISMUSP).ism/Manifest" ) do
+      insist { subject.get('path_type') } == 'ismusp'
+
+      expect(subject.get('dist_type')).to eq('vod')
+      expect(subject.get('video_type')).to eq('ism')
+      expect(subject.get('vtype')).to eq('manifest')
+      expect(subject.get('asset_id')).to eq(809735)
+      expect(subject.get('cache_key')).to eq("origin2-od.akamai.tv2.no")
+
+    end
   end
 
   describe "vod2" do
@@ -345,6 +356,19 @@ describe LogStash::Filters::Sumo do
     end
 
 
+  end
+
+  describe "ISML url  /origin1-live.akamai.tv2.no/live/727992.isml/Manifest" do
+    let(:config) do <<-CONFIG
+      filter {
+        sumo {
+          field => "pg"
+        }
+      }
+    CONFIG
+    end
+
+
     # /origin1-live.akamai.tv2.no/live/727992.isml/Manifest
     sample("pg"=>"/origin1-live.akamai.tv2.no/live/727992.isml/Manifest" ) do
       insist { subject.get('path_type') } == 'live'
@@ -352,9 +376,39 @@ describe LogStash::Filters::Sumo do
       expect(subject.get('video_type')).to eq('ism')
       expect(subject.get('asset_id')).to eq(727992)
       expect(subject.get('vtype')).to eq("manifest")
+      expect(subject.get('cache_key')).to eq("origin1-live.akamai.tv2.no")
     end
 
+    # /origin1-live.akamai.tv2.no/live/727992.isml/Manifest
+    sample("pg"=>"/origin3-hlslive.akamai.tv2.no/out/u/1285463.ism/Manifest" ) do
+      insist { subject.get('path_type') } == 'elemental live'
+      expect(subject.get('dist_type')).to eq('live')
+      expect(subject.get('video_type')).to eq('ism')
+      expect(subject.get('asset_id')).to eq(1285463)
+      expect(subject.get('vtype')).to eq("manifest")
+      expect(subject.get('cache_key')).to eq("origin3-hlslive.akamai.tv2.no")
+    end
+  end
+
+  # /stagevod/_definst_/amlst:1212305_ps97070_pd3161140.smil.ism/Manifest
+  describe "ISML url  /stagevod/_definst_/amlst:1212305_ps97070_pd3161140.smil.ism/Manifes" do
+    let(:config) do <<-CONFIG
+      filter {
+        sumo {
+          field => "pg"
+        }
+      }
+    CONFIG
+    end
+
+    sample("pg"=>" /stagevod/_definst_/amlst:1212305_ps97070_pd3161140.smil.ism/Manifest" ) do
+      insist { subject.get('path_type') } == 'ismvod'
+      expect(subject.get('dist_type')).to eq('vod')
+      expect(subject.get('video_type')).to eq('ism')
+      expect(subject.get('asset_id')).to eq(1212305)
+      expect(subject.get('vtype')).to eq("manifest")
+    end
 
   end
 
-end
+  end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-sumo
 version: !ruby/object:Gem::Version
-  version: 3.0.22
+  version: 3.0.27
 platform: ruby
 authors:
 - TV 2
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-01-22 00:00:00.000000000 Z
+date: 2018-02-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -31,9 +31,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  name: redis
+  name: logstash-devutils
   prerelease: false
-  type: :runtime
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -53,21 +53,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.1'
-- !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  name: logstash-devutils
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+description: This gem is a logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not
+  a stand-alone program
 email: leffen@gmail.com
 executables: []
 extensions: []
@@ -104,7 +92,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.8
+rubygems_version: 2.6.14
 signing_key:
 specification_version: 4
 summary: This filter extracts interessting SUMO values from the page element if present.