logstash-filter-ezproxy 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bcb21d0a739c44738fd8b3a144306a2a102dab41bae584ce309cb5d521c74dc4
-  data.tar.gz: 76afc586013a40045266fdf4fb21be58ff4aa37db1d668d3d5270d3803ed95e2
+  metadata.gz: eebd1de06645fbe8aeab87e450179bfd036476a62a9bbad3eb41dc0a68005551
+  data.tar.gz: 2e40d938da326ffb9b0c7b277f4052a5c57ed752173b91ad7c2c789961833a04
 SHA512:
-  metadata.gz: 576a44b17415f67d2f70f73f4fda6692077bf30ddc98060730969d5c108cca80f67d31d76615df7507ec0eda4bbff974a48510dec586915ae75ad460fc147174
-  data.tar.gz: 3dac55ab855093a2e9bc8ac881946ccf7036f20b4962a62b5d46566557cb528553ce26337ba2af9ef661dfa19004934f4fa18aff61e4fd63269f5b8347fa2965
+  metadata.gz: 9a622e16fff02eaed6b7267a047092d95a5d9788fc0705bdd63336859b9a8cddbb75b79e4c551d1fae90de6740a6b69f7f537ea8f48a589dfb2db5c3be378677
+  data.tar.gz: 9ba08935fa09b47a6da50d16f6acf3d66c4987f7bc09a2dff5889603286ecd3faf2b4fa77071c544ccdf560fa6de5d8b210dfd07b4904e02691ac16b12f4a9f9

data/Gemfile

@@ -2,4 +2,4 @@ source 'https://rubygems.org'
 gemspec
 # gem "logstash", :github => "elastic/logstash", :branch => "6.1"
 
-gem 'rspec', '~> 3.0'
+gem 'rspec', '~> 3.0'

data/lib/logstash/filters/cambridge.rb

@@ -0,0 +1,123 @@
+module Cambridge
+  def Cambridge.parse (path, params, uri)
+
+    url = uri.to_s
+
+    data = {
+        "provider" => "cambridge"
+    }
+
+    if (params.key?('url'))
+      url = params['url'][0]
+    end
+
+    if (params.key?('fileId') && !/\|/.match(params['fileId'][0]))
+      data['print_identifier'] = params['fileId'][0][1, 4] + '-' + params['fileId'][0][5, 4]
+      data['unit_id'] = params['fileId'][0]
+    end
+
+    if params.key?('jid')
+      data['title_id'] = params['jid'][0]
+    end
+
+    match = /\/action\/([a-z]+)/i.match(url)
+
+    if (match)
+      case (match[1])
+      when 'displayJournal'
+        data['unit_id'] = params['jid'][0]
+        data['rtype']  = 'TOC'
+        data['mime']   = 'MISC'
+
+      when 'displayJournalTab'
+        data['rtype'] = 'TOC'
+        data['mime']  = 'MISC'
+
+      when 'displayIssue'
+        data['rtype']  = 'TOC'
+        data['mime']   = 'MISC'
+        data['unit_id'] = params['iid'][0]
+        data['volume'] = params['volumeId'][0]
+        data['issue']  = params['issueId'][0]
+        
+      when 'displayFulltext'
+        if (params.key?('pdftype'))
+          data['unit_id'] = params['aid'][0]
+          data['rtype']  = 'ARTICLE'
+          data['mime']   = 'PDF'
+          data['volume'] = params['volumeId'][0]
+          data['issue']  = params['issueId'][0]
+
+        else
+          if (!data.key?('unit_id'))
+            data['unit_id'] = params['aid'][0]
+          end
+        
+          data['rtype']  = 'ARTICLE'
+          data['mime']   = 'HTML'
+          data['volume'] = params['volumeId'][0]
+          data['issue'] = params['issueId'][0]
+
+          if (params.key?('fileId'))
+            data['doi'] = '10.1017/' + params['fileId'][0]
+          end
+        end   
+        
+      when 'displayAbstract'
+        data['rtype'] = 'ABS'
+        data['mime']  = 'HTML'
+        if (params.key?('fileId'))
+          data['doi'] = '10.1017/' + params['fileId'][0]
+        end
+      
+      else
+        data['title_id'] = nil
+      end
+    
+    elsif (match = /^\/core\/services\/aop-cambridge-core\/content\/view\/[a-z0-9]+\/(([SB]?[0-9]+)[a-z0-9._-]+)\.pdf\//i.match(path))
+      data['mime']   = 'PDF'
+      data['unit_id'] = match[1]
+
+      if (/^S/i.match(match[2]))
+        data['rtype'] = 'ARTICLE'
+        data['pii'] = match[2]
+      else
+        data['rtype'] = 'BOOK_SECTION'
+        data['online_identifier'] = match[2]
+      end
+    
+
+    elsif (match = /^\/core\/journals\/([a-z-]+)\/(article|issue)\/([a-z0-9-]+)/i.match(path))
+      case (match[2])
+      when 'article'
+        data['mime']     = 'HTML'
+        data['rtype']    = 'ARTICLE'
+        data['unit_id']   = match[3].split('/')[0]
+        data['title_id'] = match[1]
+
+      when 'issue'
+        data['mime']     = 'MISC'
+        data['rtype']    = 'TOC'
+        data['unit_id']   = match[1] + '/issue/'
+        data['title_id'] = match[1]
+      end
+    
+    elsif (match = /^\/core\/books\/([a-z0-9-]+)\/[a-z0-9]+$/i.match(path))
+      data['rtype']    = 'TOC'
+      data['mime']     = 'MISC'
+      data['unit_id']   = match[1]
+      data['title_id'] = match[1]
+
+    elsif (match = /^\/core\/books\/(([a-z0-9-]+)\/[a-z0-9-]+)\/[a-z0-9]+\/core-reader$/i.match(path))
+      data['rtype']    = 'BOOK_SECTION'
+      data['mime']     = 'HTML'
+      data['unit_id']   = match[1]
+      data['title_id'] = match[2]
+    else
+      data['title_id'] = nil
+    end
+
+    return data
+
+  end
+end

data/lib/logstash/filters/dawsonera.rb

@@ -1,7 +1,4 @@
 
-require 'uri'
-require 'cgi'
-
 module DawsonEra
   def DawsonEra.parse (path, params)
 
@@ -9,8 +6,6 @@ module DawsonEra
         "provider" => "dawsonera"
     }
 
-    params = {}
-
     if (match = /^(\/abstract\/([0-9]+))$/.match(path))
       data['rtype']             = 'ABS'
       data['mime']              = 'MISC'

data/lib/logstash/filters/ebscohost.rb

@@ -20,6 +20,31 @@ module Ebscohost
         "provider" => "ebscohost"
     }
 
+    if (/^\/ebscoviewerservice/i.match(path))
+      data['rtype'] = 'EBOOK'
+    end
+
+    if (params.key?('an'))
+      data['unit_id'] = params['an'][0]
+      data['title_id'] = params['an'][0]
+    end
+
+    if (params.key?('retrievalFormat'))
+      data['mime'] = params['retrievalFormat'][0]
+    end
+    
+    artifact_params = %w(artifactId theDoid doid)
+
+    artifact_params.each do |p|
+      if params.key?(p)
+        if (param_match = /(.+)\$(.+)\$([a-zA-Z]+)/.match(params[p][0]))
+          data['unit_id'] = param_match[2]
+          data['title_id'] = param_match[2]
+          data['mime'] = param_match[3]
+        end
+      end
+    end
+
     if ((match = /^\/(ehost|eds)\/([a-z]+)(?:\/[a-z]+)?$/i.match(path)))
       category = match[2].downcase
 
@@ -100,6 +125,7 @@ module Ebscohost
       if (data['unit_id'] && data['unit_id'].downcase.start_with?('doi:'))
         data['doi'] = data['unit_id'] = data['unit_id'][4..-1]
       end
+
     end
     
     return data

data/lib/logstash/filters/emerald.rb

@@ -1,7 +1,3 @@
-
-require 'uri'
-require 'cgi'
-
 module Emerald
   def Emerald.parse (path, params)
 
@@ -40,7 +36,7 @@ module Emerald
       
       data['mime']     = 'MISC'
       data['title_id'] = match[1]
-      data['unit_id']   = 'loi/' +match[1]
+      data['unit_id']   = 'loi/' + match[1]
     elsif ((match = /^\/toc\/([a-z]+)\/([0-9]+)\/([0-9]+)/.match(path)))
       data['rtype']    = 'TOC'
       data['mime']     = 'MISC'
@@ -62,7 +58,7 @@ module Emerald
       end
     
       data['title_id'] = match[4] 
-      data['unit_id'] =data['doi']  = match[2] + '/' + match[3]
+      data['unit_id'] = data['doi']  = match[2] + '/' + match[3]
     elsif ((match = /^\/doi\/([a-z]+)\/([0-9]{2}\.[0-9]{4,5})\/([0-9]+)$/.match(path)))
     
       if (match[1] === 'abs')

data/lib/logstash/filters/ezproxy.rb

@@ -2,19 +2,28 @@
 require "logstash/filters/base"
 require "logstash/namespace"
 require_relative '../helpers/url_parser'
+require_relative '../helpers/param_helper'
+require_relative '../helpers/mime_helper'
+require_relative "./cambridge"
+require_relative "./dawsonera"
+require_relative "./ebscohost"
+require_relative "./emerald"
+require_relative "./ft"
+require_relative "./gale"
+require_relative "./heinonline"
+require_relative "./ieee"
 require_relative "./jstor"
+require_relative "./lexis_webanalytics"
 require_relative "./lexisnexis"
+require_relative "./myilibrary"
+require_relative "./proquest"
 require_relative "./sage"
-require_relative "./wiley"
 require_relative "./sciencedirect"
-require_relative "./dawsonera"
-require_relative "./tandf"
-require_relative "./emerald"
-require_relative "./ebscohost"
-require_relative "./gale"
-require_relative "./ft"
+require_relative "./scopus"
 require_relative "./springer"
-require_relative "./ieee"
+require_relative "./tandf"
+require_relative "./webofknowledge"
+require_relative "./wiley"
 require 'uri'
 require 'cgi'
 
@@ -38,9 +47,38 @@ class LogStash::Filters::Ezproxy < LogStash::Filters::Base
   # The url to be parsed by the filter
   config :url, :validate => :string, :required => true
 
-  # hosts = {
-  #   "www.jstor.org" => Jstor::parse
-  # }
+  @@hosts = {
+    "www.cambridge.org" =>            lambda { |path, params, uri| Cambridge::parse(path, params, uri) },
+    "www.dawsonera.com" =>            lambda { |path, params, uri| DawsonEra::parse(path, params) },
+    "ebscohost.com" =>                lambda { |path, params, uri| Ebscohost::parse(path, params, uri) },
+    "emeraldinsight.com" =>           lambda { |path, params, uri| Emerald::parse(path, params) },
+    "www.ft.com" =>                   lambda { |path, params, uri| FT::parse(path, params) },
+    "galegroup.com" =>                lambda { |path, params, uri| Gale::parse(path, params) },
+    "heinonline.org" =>               lambda { |path, params, uri| HeinOnline::parse(path, params) },
+    "ieee.org" =>                     lambda { |path, params, uri| IEEE::parse(path, params) },
+    "www.jstor.org" =>                lambda { |path, params, uri| Jstor::parse(path, params)},
+    "www.lexisnexis.com" =>           lambda { |path, params, uri| LexisNexis::parse(path, params) },
+    "webanalytics.lexisnexis.com" =>  lambda { |path, params, uri| LexisWebAnalytics::parse(path, params, uri) },
+    "lib.myilibrary.com" =>           lambda { |path, params, uri| MyILibrary::parse(path, params) },
+    "ebookcentral.proquest.com" =>    lambda { |path, params, uri| Proquest::parse(path, params) },
+    "journals.sagepub.com" =>         lambda { |path, params, uri| Sage::parse(path, params) },
+    "els-cdn.com" =>                  lambda { |path, params, uri|
+      return ScienceDirect::parse(path, params).merge!({ 'provider' => 'elsevier-cdn'})
+    },
+    "sciencedirect.com" =>            lambda { |path, params, uri|
+      return ScienceDirect::parse(path, params).merge!({ 'provider' => 'sciencedirect'})
+    },
+    "scopus.com" =>                   lambda { |path, params, uri| Scopus::parse(path, params) },
+    "springer.com" =>                 lambda { |path, params, uri| Springer::parse(path, params) },
+    "www.tandfonline.com" =>          lambda { |path, params, uri| TandF::parse(path, params)},
+    "thomsonreuters.com" =>           lambda { |path, params, uri|
+      return WebOfKnowledge::parse(path, params).merge!({ 'provider' => 'thomsonreuters'})
+    },
+    "webofknowledge.com" =>           lambda { |path, params, uri|
+      return WebOfKnowledge::parse(path, params).merge!({ 'provider' => 'webofknowledge'})
+    },
+    "wiley.com" =>                    lambda { |path, params, uri| Wiley::parse(path, params)}
+  }
   
 
   public
@@ -50,7 +88,16 @@ class LogStash::Filters::Ezproxy < LogStash::Filters::Base
 
   public
   def filter(event)
-    input = URI::extract(event.get(@url))[0]
+    begin
+      input = URI::extract(event.get(@url))[0]
+    rescue => e
+      puts e.message
+      puts "at"
+      puts e.backtrace.inspect
+      puts "for"
+      puts @url
+      event.tag("ezproxy_parse_failure")
+    end
 
     data = {}
 
@@ -75,36 +122,28 @@ class LogStash::Filters::Ezproxy < LogStash::Filters::Base
 
     unless uri == nil
 
-      case
-      when uri.host.include?("www.jstor.org")
-        data = Jstor::parse(path, params)
-      when uri.host.include?("www.lexisnexis.com")
-        data = LexisNexis::parse(path, params)
-      when uri.host.include?("journals.sagepub.com")
-        data = Sage::parse(path, params)
-      when uri.host.include?("wiley.com")
-        data = Wiley::parse(path, params)
-      when uri.host.include?("www.sciencedirect.com")
-        data = ScienceDirect::parse(path, params)
-      when uri.host.include?("www.dawsonera.com")
-        data = DawsonEra::parse(path, params)
-      when uri.host.include?("www.tandfonline.com")
-        data = TandF::parse(path, params)
-      when uri.host.include?("www.emeraldinsight.com")
-        data = Emerald::parse(path, params)
-      when uri.host.include?("ebscohost.com")
-        data = Ebscohost::parse(path, params, uri)
-      when uri.host.include?("els-cdn.com")
-        data = ScienceDirect::parse(path, params)
-      when uri.host.include?("springer.com")
-        data = Springer::parse(path, params)
-      when uri.host.include?("galegroup.com")
-        data = Gale::parse(path, params)
-      when uri.host.include?("ieee.org")
-        data = IEEE::parse(path, params)
+
+      begin
+        data = Mime::parse(path)
+        data.merge!(ParamHelper::parse(params))
+
+        @@hosts.each do |key, value|
+          if uri.host.include?(key)
+            data.merge!(value.call(path, params, uri))
+          end
+        end
+
+        event.tag("ezproxy_parse_success")
+      rescue => e
+        puts e.message
+        puts "at"
+        puts e.backtrace.inspect
+        puts "for"
+        puts uri
+        event.tag("ezproxy_parse_failure")
       end
+      data['path'] = path
       event.set("request_metadata", data)
-      event.tag("ezproxy_parse_success")
     else
       event.tag("ezproxy_parse_failure")
     end

data/lib/logstash/filters/gale.rb

@@ -1,6 +1,3 @@
-require 'uri'
-require 'cgi'
-
 module Gale
   def Gale.parse (path, params)
     
@@ -38,37 +35,51 @@ module Gale
     elsif (/^\/ps\/pdfViewer$/i.match(path))
       data['rtype']  = 'BOOK'
       data['mime']   = 'PDF'
-      data['unit_id'] = param.docId
+      if params.key?('docId')
+        data['unit_id'] = params['docId'][0]
+      end
 
     elsif (/^\/gdc-artemis\/bulkPdfDownload$/i.match(path))
       data['rtype']  = 'ARTICLE'
       data['mime']   = 'PDF'
-      data['unit_id'] = param.file_name
+      if params.key?('file_name')
+        data['unit_id'] = params['file_name'][0]
+      end
 
     elsif (/^\/gdc\/artemis\/ManuscriptsDetailsPage\/ManuscriptsDetailsWindow$/i.match(path))
       data['rtype']  = 'ARTICLE'
       data['mime']   = 'HTML'
-      data['unit_id'] = param.documentId
+      if params.key?('documentId')
+        data['unit_id'] = params['documentId'][0]
+      end
 
     elsif (/^\/gdsc\/retrieve.do$/i.match(path))
       data['rtype']  = 'ARTICLE'
       data['mime']   = 'HTML'
-      data['unit_id'] = param.contentSet
+      if params.key?('contentSet')
+        data['unit_id'] = params['contentSet'][0]
+      end
 
     elsif (/^\/gdsc\/downloadDocument.do$/i.match(path))
       data['rtype']  = 'ARTICLE'
       data['mime']   = 'PDF'
-      data['unit_id'] = param.docId
+      if params.key?('docId')
+        data['unit_id'] = params['docId'][0]
+      end
 
     elsif (/^\/([a-z]+)\/([a-z]+)\/MonographsDetailsPage\/MonographsDetailsWindow$/i.match(path))
       data['rtype']  = 'ARTICLE'
       data['mime']   = 'HTML'
-      data['unit_id'] = param.documentId
+      if params.key?('documentId')
+        data['unit_id'] = params['documentId'][0]
+      end
 
     elsif (/^\/([a-z]+)\/archive\/FeatureArticlesDetailsPage\/FeatureArticlesDetailsWindow$/i.match(path))
       data['rtype']  = 'ENCYCLOPAEDIA_ENTRY'
       data['mime']   = 'HTML'
-      data['unit_id'] = param.documentId
+      if params.key?('documentId')
+        data['unit_id'] = params['documentId'][0]
+      end
     end
 
     return data

data/lib/logstash/filters/heinonline.rb

@@ -0,0 +1,44 @@
+
+module HeinOnline
+  def HeinOnline.parse (path, params)
+
+    data = {
+        "provider" => "heinonline"
+    }
+
+    if (/^\/HOL\/(P.*)/.match(path))
+      data['rtype']    = 'ARTICLE'
+      data['mime']     = 'PDF'
+      if (params.key?('handle'))
+        data['title_id'] = params['handle'][0].split('/')[1]
+        data['unit_id'] = params['handle'][0] +'/'+ (params['id'][0] || "")
+        if (params.key?('number_of_pages'))
+          data['unit_id'] += '/'+ params['number_of_pages'][0]
+        end
+      end
+      
+    
+    elsif (/^\/HOL\/Index/.match(path))
+      data['rtype']    = 'TOC'
+      data['mime']     = 'HTML'
+      if (params.key?('index'))
+        data['title_id'] = params['index'][0].split('/')[1]
+        data['unit_id'] = params['index'][0]
+      end
+    
+    elsif (/^\/HOL\/.*/.match(path))
+      data['rtype']    = 'TOC'
+      data['mime']     = 'HTML'
+      if (params.key?('handle'))
+        data['title_id'] = params['handle'][0].split('/')[1]
+        data['unit_id'] = params['handle'][0] +'/'+ (params['id'][0] || "")
+        if (params.key?('number_of_pages'))
+          data['unit_id'] += '/'+ params['number_of_pages'][0]
+        end
+      end
+    end
+    
+    return data
+
+  end
+end