logstash-filter-ezproxy 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eebd1de06645fbe8aeab87e450179bfd036476a62a9bbad3eb41dc0a68005551
-  data.tar.gz: 2e40d938da326ffb9b0c7b277f4052a5c57ed752173b91ad7c2c789961833a04
+  metadata.gz: 8a86f6f54351a599519f27e04c0b1ec9bbe9a6a8c802dac09634d86476c212ef
+  data.tar.gz: 8fea5f5da0e5da09b6b7c750f81fec7347708e994748b0a72361e8791ff0fe56
 SHA512:
-  metadata.gz: 9a622e16fff02eaed6b7267a047092d95a5d9788fc0705bdd63336859b9a8cddbb75b79e4c551d1fae90de6740a6b69f7f537ea8f48a589dfb2db5c3be378677
-  data.tar.gz: 9ba08935fa09b47a6da50d16f6acf3d66c4987f7bc09a2dff5889603286ecd3faf2b4fa77071c544ccdf560fa6de5d8b210dfd07b4904e02691ac16b12f4a9f9
+  metadata.gz: 2fd7d975bce60293bf1ede3d0c78f294fd25e94836d5a29d36c63d8e5eb83f81ad6137e88a9c018054f97814a1da0da48fa675092175c87ee5820d3c2ca0d62d
+  data.tar.gz: 6889ce78ced9789480e1d2962743c74cca3b26b5cebfbc693b40e91ddf2821c3d14f98c90f62666c9f9007a7e43da72ed223778be68f5c44c50840bb5ae5f14e

data/CONTRIBUTORS

@@ -2,7 +2,7 @@ The following is a list of people who have contributed ideas, code, bug
 reports, or in general have helped logstash along its way.
 
 Contributors:
-* Dom Belcher - dominic.belcher@gmail.com
+* Dom Belcher - d.belcher@lancaster.ac.uk
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/lib/logstash/filters/ezproxy.rb

@@ -1,32 +1,12 @@
 # encoding: utf-8
 require "logstash/filters/base"
 require "logstash/namespace"
-require_relative '../helpers/url_parser'
-require_relative '../helpers/param_helper'
-require_relative '../helpers/mime_helper'
-require_relative "./cambridge"
-require_relative "./dawsonera"
-require_relative "./ebscohost"
-require_relative "./emerald"
-require_relative "./ft"
-require_relative "./gale"
-require_relative "./heinonline"
-require_relative "./ieee"
-require_relative "./jstor"
-require_relative "./lexis_webanalytics"
-require_relative "./lexisnexis"
-require_relative "./myilibrary"
-require_relative "./proquest"
-require_relative "./sage"
-require_relative "./sciencedirect"
-require_relative "./scopus"
-require_relative "./springer"
-require_relative "./tandf"
-require_relative "./webofknowledge"
-require_relative "./wiley"
 require 'uri'
 require 'cgi'
 
+Dir[File.dirname(__FILE__) + '/../helpers/*.rb'].each { |file| require file }
+Dir[File.dirname(__FILE__) + '/./ezproxy_providers/*.rb'].each { |file| require file }
+
 # This  filter will replace the contents of the default 
 # message field with whatever you specify in the configuration.
 #
@@ -38,7 +18,11 @@ class LogStash::Filters::Ezproxy < LogStash::Filters::Base
   #
   # filter {
   #    {
-  #     message => "My message..."
+  #     url => "Field containing url to parse"
+  #     target => "Field to output metadata to - default is request_metadata"
+  #     mime_tag => true|false set inital mime types for all requests based on file extensions - default false
+  #     doc_id_tag => true|false set unit and title id from docID like params - default false
+  #     path_tag => true|false add request path to metadata - default false
   #   }
   # }
   #
@@ -46,38 +30,46 @@ class LogStash::Filters::Ezproxy < LogStash::Filters::Base
   
   # The url to be parsed by the filter
   config :url, :validate => :string, :required => true
+  config :target, :validate => :string, :default => "request_metadata"
+  config :mime_tag, :validate => :boolean, :default => false
+  config :doc_id_tag, :validate => :boolean, :default => false
+  config :path_tag, :validate => :boolean, :default => false
 
   @@hosts = {
-    "www.cambridge.org" =>            lambda { |path, params, uri| Cambridge::parse(path, params, uri) },
-    "www.dawsonera.com" =>            lambda { |path, params, uri| DawsonEra::parse(path, params) },
-    "ebscohost.com" =>                lambda { |path, params, uri| Ebscohost::parse(path, params, uri) },
-    "emeraldinsight.com" =>           lambda { |path, params, uri| Emerald::parse(path, params) },
-    "www.ft.com" =>                   lambda { |path, params, uri| FT::parse(path, params) },
-    "galegroup.com" =>                lambda { |path, params, uri| Gale::parse(path, params) },
-    "heinonline.org" =>               lambda { |path, params, uri| HeinOnline::parse(path, params) },
-    "ieee.org" =>                     lambda { |path, params, uri| IEEE::parse(path, params) },
-    "www.jstor.org" =>                lambda { |path, params, uri| Jstor::parse(path, params)},
-    "www.lexisnexis.com" =>           lambda { |path, params, uri| LexisNexis::parse(path, params) },
-    "webanalytics.lexisnexis.com" =>  lambda { |path, params, uri| LexisWebAnalytics::parse(path, params, uri) },
-    "lib.myilibrary.com" =>           lambda { |path, params, uri| MyILibrary::parse(path, params) },
-    "ebookcentral.proquest.com" =>    lambda { |path, params, uri| Proquest::parse(path, params) },
-    "journals.sagepub.com" =>         lambda { |path, params, uri| Sage::parse(path, params) },
+    "www.cambridge.org" =>            lambda { |path, params, uri| EzproxyProviders::Cambridge.parse(path, params, uri) },
+    "www.dawsonera.com" =>            lambda { |path, params, uri| EzproxyProviders::DawsonEra::parse(path, params) },
+    "ebookcentral.proquest.com" =>    lambda { |path, params, uri| EzproxyProviders::EBookCentral::parse(path, params) },
+    "ebscohost.com" =>                lambda { |path, params, uri| EzproxyProviders::Ebscohost::parse(path, params, uri) },
+    "elsevierelibrary.co.uk" =>       lambda { |path, params, uri| EzproxyProviders::Elsevier::parse(path, params) },
+    "emeraldinsight.com" =>           lambda { |path, params, uri| EzproxyProviders::Emerald::parse(path, params) },
+    "www.ft.com" =>                   lambda { |path, params, uri| EzproxyProviders::FT::parse(path, params) },
+    "galegroup.com" =>                lambda { |path, params, uri| EzproxyProviders::Gale::parse(path, params) },
+    "heinonline.org" =>               lambda { |path, params, uri| EzproxyProviders::HeinOnline::parse(path, params) },
+    "ieee.org" =>                     lambda { |path, params, uri| EzproxyProviders::IEEE::parse(path, params) },
+    "www.jstor.org" =>                lambda { |path, params, uri| EzproxyProviders::Jstor::parse(path, params) },
+    "www.lexisnexis.com" =>           lambda { |path, params, uri| EzproxyProviders::LexisNexis::parse(path, params) },
+    "webanalytics.lexisnexis.com" =>  lambda { |path, params, uri| EzproxyProviders::LexisWebAnalytics::parse(path, params, uri) },
+    "lib.myilibrary.com" =>           lambda { |path, params, uri| EzproxyProviders::MyILibrary::parse(path, params) },
+    "www.oxfordscholarship.com" =>    lambda { |path, params, uri| EzproxyProviders::Oxford::parse(path, params) },
+    "search.proquest.com" =>          lambda { |path, params, uri| EzproxyProviders::ProquestSearch::parse(path, params) },
+    "journals.sagepub.com" =>         lambda { |path, params, uri| EzproxyProviders::Sage::parse(path, params) },
     "els-cdn.com" =>                  lambda { |path, params, uri|
-      return ScienceDirect::parse(path, params).merge!({ 'provider' => 'elsevier-cdn'})
+      return EzproxyProviders::ScienceDirect::parse(path, params).merge!({ 'provider' => 'elsevier-cdn' })
     },
     "sciencedirect.com" =>            lambda { |path, params, uri|
-      return ScienceDirect::parse(path, params).merge!({ 'provider' => 'sciencedirect'})
+      return EzproxyProviders::ScienceDirect::parse(path, params).merge!({ 'provider' => 'sciencedirect' })
     },
-    "scopus.com" =>                   lambda { |path, params, uri| Scopus::parse(path, params) },
-    "springer.com" =>                 lambda { |path, params, uri| Springer::parse(path, params) },
-    "www.tandfonline.com" =>          lambda { |path, params, uri| TandF::parse(path, params)},
+    "scopus.com" =>                   lambda { |path, params, uri| EzproxyProviders::Scopus::parse(path, params) },
+    "springer.com" =>                 lambda { |path, params, uri| EzproxyProviders::Springer::parse(path, params) },
+    "www.tandfonline.com" =>          lambda { |path, params, uri| EzproxyProviders::TandF::parse(path, params) },
     "thomsonreuters.com" =>           lambda { |path, params, uri|
-      return WebOfKnowledge::parse(path, params).merge!({ 'provider' => 'thomsonreuters'})
+      return EzproxyProviders::WebOfKnowledge::parse(path, params).merge!({ 'provider' => 'thomsonreuters' })
     },
     "webofknowledge.com" =>           lambda { |path, params, uri|
-      return WebOfKnowledge::parse(path, params).merge!({ 'provider' => 'webofknowledge'})
+      return EzproxyProviders::WebOfKnowledge::parse(path, params).merge!({ 'provider' => 'webofknowledge' })
     },
-    "wiley.com" =>                    lambda { |path, params, uri| Wiley::parse(path, params)}
+    "westlaw.co.uk" =>                lambda { |path, params, uri| EzproxyProviders::Westlaw::parse(path, params) },
+    "wiley.com" =>                    lambda { |path, params, uri| EzproxyProviders::Wiley::parse(path, params) }
   }
   
 
@@ -106,26 +98,15 @@ class LogStash::Filters::Ezproxy < LogStash::Filters::Base
     path = parsed_url['path']
     params = parsed_url['params']
 
-    # if (uri.host == "ezproxy.lancs.ac.uk")
-    #   if (uri.query)
-    #     puts uri
-    #     params = CGI::parse(uri.query)
-    #     if params.key?('url')
-    #       uri = URI(params['url'][0])
-    #     elsif params.key?('qurl')
-    #       uri = URI(params['qurl'][0])
-    #     end
-    #     event.tag("requested_host_ezproxy")
-    #     event.set("requested_host", uri.host)
-    #   end
-    # end
-
     unless uri == nil
 
-
       begin
-        data = Mime::parse(path)
-        data.merge!(ParamHelper::parse(params))
+        if @mime_tag
+          data.merge!(Mime::parse(path))
+        end
+        if @doc_id_tag
+          data.merge!(ParamHelper::parse(params))
+        end
 
         @@hosts.each do |key, value|
           if uri.host.include?(key)
@@ -142,8 +123,10 @@ class LogStash::Filters::Ezproxy < LogStash::Filters::Base
         puts uri
         event.tag("ezproxy_parse_failure")
       end
-      data['path'] = path
-      event.set("request_metadata", data)
+      if @path_tag
+        data['path'] = path
+      end
+      event.set(@target, data)
     else
       event.tag("ezproxy_parse_failure")
     end

data/lib/logstash/filters/ezproxy_providers.rb

@@ -0,0 +1 @@
+module EzproxyProviders end

data/lib/logstash/filters/{cambridge.rb → ezproxy_providers/cambridge.rb}

@@ -1,5 +1,7 @@
-module Cambridge
-  def Cambridge.parse (path, params, uri)
+require_relative '../ezproxy_providers'
+
+class EzproxyProviders::Cambridge
+  def self.parse (path, params, uri)
 
     url = uri.to_s
 
@@ -120,4 +122,4 @@ module Cambridge
     return data
 
   end
-end
+end

data/lib/logstash/filters/{dawsonera.rb → ezproxy_providers/dawsonera.rb}

@@ -1,6 +1,7 @@
+require_relative '../ezproxy_providers'
 
-module DawsonEra
-  def DawsonEra.parse (path, params)
+class EzproxyProviders::DawsonEra
+  def self.parse (path, params)
 
     data = {
         "provider" => "dawsonera"

data/lib/logstash/filters/{proquest.rb → ezproxy_providers/ebookcentral.rb}

@@ -1,5 +1,7 @@
-module Proquest
-  def Proquest.parse (path, params)
+require_relative '../ezproxy_providers'
+
+class EzproxyProviders::EBookCentral
+  def self.parse (path, params)
     data = {
       "provider" => "proquest"
     }

data/lib/logstash/filters/{ebscohost.rb → ezproxy_providers/ebscohost.rb}

@@ -1,8 +1,9 @@
 
 require 'uri'
 require 'cgi'
+require_relative '../ezproxy_providers'
 
-module Ebscohost
+class EzproxyProviders::Ebscohost
 
   @openUrlFields = {
     'issn' => 'print_identifier',
@@ -14,7 +15,7 @@ module Ebscohost
     'id' => 'unit_id'
   }
 
-  def Ebscohost.parse (path, params, uri)
+  def self.parse (path, params, uri)
 
     data = {
         "provider" => "ebscohost"

data/lib/logstash/filters/ezproxy_providers/elsevier.rb

@@ -0,0 +1,18 @@
+require_relative '../ezproxy_providers'
+
+class EzproxyProviders::Elsevier
+  def self.parse (path, params)
+
+    data = {
+        "provider" => "elsevier"
+    }
+
+    if match = /\/(product|pdfreader)\/([a-z0-9\-]+)/i.match(path)
+      data['unit_id'] = match[2]
+      data['title_id'] = match[2]
+    end
+
+    return data
+
+  end
+end

data/lib/logstash/filters/{emerald.rb → ezproxy_providers/emerald.rb}

@@ -1,5 +1,7 @@
-module Emerald
-  def Emerald.parse (path, params)
+require_relative '../ezproxy_providers'
+
+class EzproxyProviders::Emerald
+  def self.parse (path, params)
 
     data = {
       "provider" => "emerald"

data/lib/logstash/filters/{ft.rb → ezproxy_providers/ft.rb}

@@ -1,7 +1,7 @@
-require "uri"
+require_relative '../ezproxy_providers'
 
-module FT
-  def FT.parse (path, params)
+class EzproxyProviders::FT
+  def self.parse (path, params)
     
     data = {
       "provider" => "ft"

data/lib/logstash/filters/{gale.rb → ezproxy_providers/gale.rb}

@@ -1,5 +1,7 @@
-module Gale
-  def Gale.parse (path, params)
+require_relative '../ezproxy_providers'
+
+class EzproxyProviders::Gale
+  def self.parse (path, params)
     
     data = {
       "provider" => "gale"

data/lib/logstash/filters/{heinonline.rb → ezproxy_providers/heinonline.rb}

@@ -1,6 +1,7 @@
+require_relative '../ezproxy_providers'
 
-module HeinOnline
-  def HeinOnline.parse (path, params)
+class EzproxyProviders::HeinOnline
+  def self.parse (path, params)
 
     data = {
         "provider" => "heinonline"

data/lib/logstash/filters/{ieee.rb → ezproxy_providers/ieee.rb}

@@ -1,8 +1,7 @@
-require 'uri'
-require 'cgi'
+require_relative '../ezproxy_providers'
 
-module IEEE
-  def IEEE.parse (path, params)
+class EzproxyProviders::IEEE
+  def self.parse (path, params)
 
     data = {
       "provider" => "springer"

data/lib/logstash/filters/{jstor.rb → ezproxy_providers/jstor.rb}

@@ -1,6 +1,7 @@
+require_relative '../ezproxy_providers'
 
-module Jstor
-    def Jstor.parse (path, params)
+class EzproxyProviders::Jstor
+    def self.parse (path, params)
 
         data = {
             "provider" => "jstor"

data/lib/logstash/filters/{lexis_webanalytics.rb → ezproxy_providers/lexis_webanalytics.rb}

@@ -1,7 +1,8 @@
 require 'json'
+require_relative '../ezproxy_providers'
 
-module LexisWebAnalytics
-  def LexisWebAnalytics.parse (path, params, uri)
+class EzproxyProviders::LexisWebAnalytics
+  def self.parse (path, params, uri)
 
     raw_url = uri.to_s
 

data/lib/logstash/filters/{lexisnexis.rb → ezproxy_providers/lexisnexis.rb}

@@ -1,7 +1,7 @@
-require 'uri'
+require_relative '../ezproxy_providers'
 
-module LexisNexis
-  def LexisNexis.parse (path, params)
+class EzproxyProviders::LexisNexis
+  def self.parse (path, params)
 
     data = {
       "provider" => "lexisnexis"

data/lib/logstash/filters/{myilibrary.rb → ezproxy_providers/myilibrary.rb}

@@ -1,5 +1,7 @@
-module MyILibrary
-  def MyILibrary.parse (path, params)
+require_relative '../ezproxy_providers'
+
+class EzproxyProviders::MyILibrary
+  def self.parse (path, params)
 
     data = {
       "provider" => "myilibrary"
@@ -15,7 +17,9 @@ module MyILibrary
 
     elsif (/\/Viewer\/getImage\_Servlet\.aspx/i.match(path))
       data['rtype'] = 'BOOK_PAGE'
-      data['mime'] = 'JPG'
+      if params.key?('codec')
+        data['mime'] = params['codec'][0].upcase
+      end
 
     elsif (/\/Viewer\/get[DP]MP\_Servlet\.aspx/i.match(path))
       data['rtype'] = 'BOOK_PAGE'

data/lib/logstash/filters/{oxford.rb → ezproxy_providers/oxford.rb}

@@ -1,5 +1,7 @@
-module Oxford
-  def Oxford.parse (path, params)
+require_relative '../ezproxy_providers'
+
+class EzproxyProviders::Oxford
+  def self.parse (path, params)
 
     data = {
       "provider" => "oxford"
@@ -25,7 +27,7 @@ module Oxford
         data['rtype'] = 'BOOK'
       end
 
-      if params.key?('print')
+      if params.key?('print') && !params['print'][0].nil?
         data['mime'] = params['print'][0].upcase
       end
 

data/lib/logstash/filters/ezproxy_providers/proquest-search.rb

@@ -0,0 +1,27 @@
+require_relative '../ezproxy_providers'
+
+class EzproxyProviders::ProquestSearch
+  def self.parse (path, params)
+
+    data = {
+      "provider" => "proquest"
+    }
+
+    if params.key?('t:ac')
+      data['unit_id'] = params['t:ac'][0]
+      data['title_id'] = params['t:ac'][0]
+    end
+
+    if (match = /\/docview\/([0-9]+)\//i.match(path))
+      data['unit_id'] = match[1]
+      data['title_id'] = match[1]
+
+      if /fulltextpdf/i.match(path)
+        data['mime'] = 'PDF'
+      end
+    end
+
+    return data
+
+  end
+end

data/lib/logstash/filters/{sage.rb → ezproxy_providers/sage.rb}

@@ -1,7 +1,7 @@
-require "uri"
+require_relative '../ezproxy_providers'
 
-module Sage
-  def Sage.parse (path, params)
+class EzproxyProviders::Sage
+  def self.parse (path, params)
     
     data = {
       "provider" => "sage"

data/lib/logstash/filters/{sciencedirect.rb → ezproxy_providers/sciencedirect.rb}

@@ -1,8 +1,7 @@
-require 'uri'
-require 'cgi'
+require_relative '../ezproxy_providers'
 
-module ScienceDirect
-  def ScienceDirect.parse (path, params)
+class EzproxyProviders::ScienceDirect
+  def self.parse (path, params)
 
     data = {}