logstash-filter-ezproxy 0.1.0 → 0.1.1

data/lib/logstash/filters/lexisnexis.rb

@@ -1,35 +1,30 @@
 require 'uri'
 
 module LexisNexis
-  def LexisNexis.parse (input)
-    uri = URI(URI.unescape(input))
-
-    path = uri.path
+  def LexisNexis.parse (path, params)
 
     data = {
       "provider" => "lexisnexis"
     }
 
     if (/\/droit\/results\/docview\/docview/.match(path))
-      if (uri.query)
-        params = CGI::parse(uri.query)
 
-        if (params["risb"])
-          data["title_id"] = params["risb"][0]
-          data["unit_id"] = params["risb"][0]
-        end
+      if (params.key?("risb"))
+        data["title_id"] = params["risb"][0]
+        data["unit_id"] = params["risb"][0]
+      end
 
-        if (params["format"])
-          case params["format"][0]
-            when 'GNBFULL'
-              data["rtype"] = 'ARTICLE'
-              data["mime"] = 'HTML'
-            when 'AUTRECAS' 
-              data["rtype"] = 'ARTICLE'
-              data["mime"] = 'HTML'
-          end
+      if (params.key?("format"))
+        case params["format"][0]
+          when 'GNBFULL'
+            data["rtype"] = 'ARTICLE'
+            data["mime"] = 'HTML'
+          when 'AUTRECAS' 
+            data["rtype"] = 'ARTICLE'
+            data["mime"] = 'HTML'
         end
       end
+      
     end
     
     return data

data/lib/logstash/filters/sage.rb

@@ -1,11 +1,8 @@
 require "uri"
 
 module Sage
-  def Sage.parse (input)
-    uri = URI(URI.unescape(input))
-
-    path = uri.path
-
+  def Sage.parse (path, params)
+    
     data = {
       "provider" => "sage"
     }

data/lib/logstash/filters/sciencedirect.rb

@@ -1,15 +1,8 @@
 require 'uri'
+require 'cgi'
 
 module ScienceDirect
-  def ScienceDirect.parse (input)
-    uri = URI(URI.unescape(input))
-
-    path = uri.path
-    params = {}
-
-    if (uri.query)
-      params = CGI::parse(uri.query)
-    end
+  def ScienceDirect.parse (path, params)
 
     data = {
       "provider" => "sciencedirect"
@@ -30,7 +23,7 @@ module ScienceDirect
         data["pii"]    = (params['_hubEid'][0] || '').split('-')[2]
 
 
-        if (params.key?("_isbn")|| params.key?('isBook'))
+        if (params.key?("_isbn") || params.key?('isBook'))
           data['rtype']            = 'CHAPTERS_BUNDLE'
           data['print_identifier'] = params['_isbn'][0]
           data['title_id']         = params['_isbn'][0]

data/lib/logstash/filters/springer.rb

@@ -0,0 +1,142 @@
+require 'uri'
+require 'cgi'
+
+module Springer
+  def Springer.parse (path, params)
+
+    data = {
+      "provider" => "springer"
+    }
+
+    if ((match = /\/journal(\/volumesAndIssues)?\/([0-9]+)/.match(path)))
+      data['title_id'] = match[2]
+      data['unit_id']   = match[2]
+      data['rtype']    = 'TOC'
+      data['mime']     = 'MISC'
+
+    elsif ((match = /^\/(article|book|protocol)\/([0-9]+\.[0-9]+\/[^\/]+)(\/page\/[0-9]+)?(\/fulltext.html)?/.match(path)))
+      data['doi']    = match[2]
+      data['unit_id'] = match[2].split('/')[1] + (match[3] || '')
+
+      case (match[1])
+      when 'article'
+        data['rtype'] = 'ARTICLE'
+        data['mime']  = 'HTML'
+      when 'book'
+        data['rtype'] = 'BOOK'
+        data['mime']  = 'HTML'
+        
+        if (/^\/book\/([0-9]+\.[0-9]+\/([0-9-])+)$/.match(path))
+          data['rtype']             = 'TOC'
+          data['mime']              = 'MISC'
+          data['online_identifier'] = match[2].split('/')[1]
+        end
+
+      when 'protocol'
+        data['rtype'] = 'BOOK'
+        data['mime']  = 'HTML'
+      end
+
+    elsif ((match = /^\/content\/pdf\/(10\.[0-9]+\/(.+?))(\.pdf)?$/.match(path)))
+
+      data['doi']    = match[1]
+      data['unit_id'] = match[2]
+      data['rtype']  = 'ARTICLE'
+      data['mime']   = 'PDF'
+
+      if (/^(\d-*){13}(?![\d-])/.match(match[2]))
+        data['rtype'] = 'BOOK'
+      end
+
+
+    elsif ((match = /^\/content\/([0-9]{4}-[0-9]{4})/.match(path)))
+      data['print_identifier'] = match[1]
+      data['unit_id']           = match[1]
+      data['rtype']            = 'TOC'
+      data['mime']             = 'MISC'
+
+    elsif ((match = /^\/content\/([a-zA-Z0-9]+)(\/fulltext.pdf)?/.match(path)))
+      data['unit_id'] = match[1]
+      data['rtype']  = 'ABS'
+      data['mime']   = 'MISC'
+
+    elsif ((match = /^\/chapter\/(([0-9]+\.[0-9]+)\/([^\/]*))(\/([a-z]+)\.html)?/.match(path)))
+      data['doi']    = match[1]
+      data['unit_id'] = match[3]
+      data['rtype']  = 'BOOK_SECTION'
+      data['mime']   = 'HTML'
+
+    elsif ((match = /^\/(book)?series\/([0-9]+)/.match(path)))
+      data['title_id'] = match[2]
+      data['unit_id']   = match[2]
+      data['rtype']    = 'BOOKSERIE'
+      data['mime']     = 'MISC'
+
+    elsif ((match = /^\/openurl.asp/.match(path)))
+      if (params.key?('genre') && params['genre'][0] == 'journal')
+        if (params['issn'][0])
+          data['print_identifier'] = params['issn'][0]
+          data['unit_id'] = params['issn'][0]
+        end
+
+        data['rtype'] = 'TOC'
+        data['mime']  = 'MISC'
+      end
+
+    elsif ((match = /^\/static\/pdf\/([0-9]+)\/([a-zA-Z]{3})([^\/]+)\.pdf/.match(path)))
+      if ((params.key('ext') && params['ext'] == '.pdf') || params['token2'])
+        data['title_id'] = match[1]
+        data['mime'] = 'PDF'
+        data['unit_id'] = URI::unescape(match[3])[1..-1]
+
+        case match[2]
+        when 'art'
+          data['unit_id'] = data['unit_id'].split('/')[1]
+          data['doi']   = URI::unescape(match[3])[1..-1]
+          data['rtype'] = 'ARTICLE'
+
+        when 'chp'
+          data['unit_id'] = data['unit_id'].split('/')[1]
+          data['doi']   = URI::unescape(match[3])[1..-1]
+          data['rtype'] = 'BOOK_SECTION'
+
+        when 'bok'
+          data['online_identifier'] = data['unit_id']
+          data['rtype'] = 'BOOK'
+
+        when 'bfm'
+          data['online_identifier'] = data['unit_id'].split('/')[0]
+          data['rtype'] = 'TOC'
+
+        else
+          data['rtype'] = 'TOC'
+        end
+      end
+
+
+    elsif ((match = /^\/(download|static)\/([a-z]+)\/(([0-9.]*)\/([^\/]*)).epub/.match(path)))
+      if (/([0-9]+)\.([0-9]+)/.test(match[4]))
+        data['doi']      = match[3]
+        data['unit_id']   = match[5] + '.epub'
+        data['print_identifier'] = match[5]
+      else
+        if (testunitid = /([\w%]*)[A-Z]{1}([0-9-]+)/.match(match[5]))
+          data['unit_id']   = testunitid[2] + '.epub'
+          data['print_identifier'] = testunitid[2]
+        end
+      end
+
+      data['rtype']    = 'BOOK'
+      data['mime']     = 'EPUB'
+    end
+
+    if (data['doi'])
+      title_id = /s([0-9]+)-/.match(data['doi'])
+      if (title_id && title_id[1])
+        data['title_id'] = '' + Integer(title_id[1], 10).to_s;
+      end
+    end  
+
+    return data;
+  end
+end

data/lib/logstash/filters/tandf.rb

@@ -3,15 +3,7 @@ require 'uri'
 require 'cgi'
 
 module TandF
-  def TandF.parse (input)
-
-    uri = URI(URI.unescape(input))
-
-    path = uri.path
-    params = {}
-    if (uri.query)
-        params = CGI::parse(uri.query)
-    end
+  def TandF.parse (path, params)
 
     data = {
         "provider" => "tandf"

data/lib/logstash/filters/wiley.rb

@@ -1,16 +1,11 @@
 require 'uri'
 
 module Wiley
-  def Wiley.parse (input)
-    uri = URI(URI.unescape(input))
-
-    path = uri.path
+  def Wiley.parse (path, params)
 
     data = {
       "provider" => "wiley"
     }
-
-    #!/usr/bin/env node
         
     if ((match = /\/journal\/(10\.[0-9]+\/(\(ISSN\)([0-9]{4}-[0-9]{3}[0-9xX])))/i.match(path)))
       data['doi']    = match[1];
@@ -27,7 +22,7 @@ module Wiley
       data['rtype']    = 'TOC';
       data['mime']     = 'MISC';
 
-    data['publication_date'] = match[4];
+      data['publication_date'] = match[4];
 
     elsif ((match = /^\/doi\/(10\.[0-9]+\/(j\.([0-9]{4}-[0-9]{3}[0-9xX])\.([0-9]{4})\.[^.]+\.[^.]+))\/abstract$/i.match(path)))
       data['doi']    = match[1];
@@ -35,8 +30,8 @@ module Wiley
       data['rtype']  = 'ABS';
       data['mime']   = 'MISC';
 
-    data['online_identifier'] = match[3];
-    data['publication_date']  = match[4];
+      data['online_identifier'] = match[3];
+      data['publication_date']  = match[4];
 
     elsif ((match = /^\/doi\/(10\.[0-9]+\/(([^.]+)\.([0-9]{4})[0-9]+))\/abstract$/i.match(path)))
       data['doi']      = match[1];
@@ -45,7 +40,7 @@ module Wiley
       data['rtype']    = 'ABS';
       data['mime']     = 'MISC';
 
-    data['publication_date'] = match[4];
+      data['publication_date'] = match[4];
 
     elsif ((match = /^\/doi\/(10\.[0-9]+\/(([^.]+)\.[0-9]+))\/full$/i.match(path)))
       data['doi']      = match[1];
@@ -116,18 +111,15 @@ module Wiley
       data['rtype'] = 'ARTICLE';
       data['mime']  = 'READCUBE';
 
-      if (uri.query)
-        params = CGI::parse(uri.query)
-
-        if (params["resource"])
+      if (params.key?("resource"))
 
-          data['doi']    = params["resource"][0];
-          data['unit_id'] = params["resource"][0].split('/')[1];
-          if ((match = /(10\.[0-9]+)\/([0-9]{4})([a-z0-9]{2})([^\/]+)$/i.match(params['resource'][0])))
-              data['title_id'] = match[3].upcase;
-          end
+        data['doi']    = params["resource"][0];
+        data['unit_id'] = params["resource"][0].split('/')[1];
+        if ((match = /(10\.[0-9]+)\/([0-9]{4})([a-z0-9]{2})([^\/]+)$/i.match(params['resource'][0])))
+            data['title_id'] = match[3].upcase;
         end
       end
+      
 
 
     elsif ((match = /^\/doi\/(10\.[0-9]+\/([a-z]{1}[0-9]{8}([0-9]{2})[a-z0-9]+))\/pdf$/i.match(path)))

data/lib/logstash/helpers/url_parser.rb

@@ -0,0 +1,23 @@
+require 'uri'
+require 'cgi'
+
+module URLParser
+  def URLParser.parse (input)
+    begin
+      uri = URI(input)
+      path = URI::unescape(uri.path)
+      params = {}
+      if (uri.query)
+          params = CGI::parse(URI::unescape(uri.query))
+      end
+      return {
+        "uri" => uri,
+        "path" => path,
+        "params" => params
+      }
+    rescue
+      puts input
+      return {}
+    end
+  end
+end

data/logstash-filter-ezproxy.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-filter-ezproxy'
-  s.version       = '0.1.0'
+  s.version       = '0.1.1'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'Write a short summary, because Rubygems requires one.'
   s.authors       = ['Dom Belcher']
@@ -16,6 +16,7 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
+  #s.add_dependency "logstash-core", "~> 6.1.0"
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/filters/dawsonera/dawsonera_spec.rb

@@ -1,11 +1,16 @@
 require 'logstash/filters/dawsonera'
 require 'csv'
+require 'logstash/helpers/url_parser'
 
 RSpec.describe DawsonEra do
     CSV.foreach('spec/filters/dawsonera/dawsonera.2014-09-03.csv', { :col_sep => ';', headers: true }) do |row|
+        parsed_url = URLParser::parse(row[4])
+        path = parsed_url['path']
+        params = parsed_url['params']
+
         name = "tests a URL " + row[4]
         it name do
-            data = DawsonEra.parse(row[4])
+            data = DawsonEra.parse(path, params)
             expect(data["unit_id"]).to eq(row[0])
             expect(data["online_identifier"]).to eq(row[1])
             expect(data["rtype"]).to eq(row[2])

data/spec/filters/ebscohost/ebscohost_spec.rb

@@ -1,11 +1,17 @@
 require 'logstash/filters/ebscohost'
+require 'logstash/helpers/url_parser'
 require 'csv'
 
 RSpec.describe Ebscohost do
     CSV.foreach('spec/filters/ebscohost/ebscohost.2014-08-21.csv', { :col_sep => ';', headers: true }) do |row|
+        parsed_url = URLParser::parse(row[11])
+        uri = parsed_url['uri']
+        path = parsed_url['path']
+        params = parsed_url['params']
+
         name = "tests a URL " + row[11]
         it name do
-            data = Ebscohost.parse(row[11])
+            data = Ebscohost.parse(path, params, uri)
             expect(data["unit_id"]).to eq(row[0])
             expect(data["rtype"]).to eq(row[1])
             expect(data["mime"]).to eq(row[2])

data/spec/filters/emerald/emerald_spec.rb

@@ -1,11 +1,16 @@
 require 'logstash/filters/emerald'
+require 'logstash/helpers/url_parser'
 require 'csv'
 
 RSpec.describe Emerald do
     CSV.foreach('spec/filters/emerald/emerald.2015-08-11.csv', { :col_sep => ';', headers: true }) do |row|
+        parsed_url = URLParser::parse(row[6])
+        path = parsed_url['path']
+        params = parsed_url['params']
+
         name = "tests a URL " + row[6]
         it name do
-            data = Emerald.parse(row[6])
+            data = Emerald.parse(path, params)
             expect(data["title_id"]).to eq(row[0])
             expect(data["doi"]).to eq(row[1])
             expect(data["unit_id"]).to eq(row[2])

data/spec/filters/ft/ft.2016-11-22.csv

@@ -0,0 +1,9 @@
+out-title_id;out-unitid;out-publication_date;out-rtype;out-mime;in-url
+;0b4a4790-6454-11e6-8310-ecf0bddad227;;ARTICLE;HTML;http://www.ft.com/cms/s/0/0b4a4790-6454-11e6-8310-ecf0bddad227.html#axzz4HgTPkTq0
+5088258522001;Market-Minute-Federal-Reserve-in-focus;;VIDEO;MISC;http://video.ft.com/5088258522001/Market-Minute-Federal-Reserve-in-focus/Editors-Choice
+2172893;juicing-the-numbers-is-ok-if-youre-in-silicon-valley-apparently;2016;ARTICLE;HTML;http://ftalphaville.ft.com/2016/08/18/2172893/juicing-the-numbers-is-ok-if-youre-in-silicon-valley-apparently/
+;;;ARTICLE;HTML;http://digital.olivesoftware.com/Olive/ODE/FTePaperUK/?simigvis=OC4zMzguODQzMzc4ODU1NTI4MDQuMTQ3MTUyMTE1ODI3My43ZGE0ZWZkZQ__*
+;;;TOOL;HTML;http://markets.ft.com/data/portfolio/dashboard
+;emerging-voices;;REPORT;HTML;http://www.ft.com/reports/emerging-voices
+;0b5de310-6b56-11e5-8171-ba1968cf791a;;REPORT;PDF;http://im.ft-static.com/content/images/0b5de310-6b56-11e5-8171-ba1968cf791a.pdf
+;global-mba-ranking-2016;2016;TOOL;HTML;http://rankings.ft.com/businessschoolrankings/mcgill-university/global-mba-ranking-2016#global-mba-ranking-2016

data/spec/filters/ft/ft_spec.rb

@@ -0,0 +1,21 @@
+require 'logstash/filters/ft'
+require 'logstash/helpers/url_parser'
+require 'csv'
+
+RSpec.describe FT do
+    CSV.foreach('spec/filters/ft/ft.2016-11-22.csv', { :col_sep => ';', headers: true }) do |row|
+        parsed_url = URLParser::parse(row[5])
+        path = parsed_url['path']
+        params = parsed_url['params']
+
+        name = "tests a URL " + row[5]
+        it name do
+            data = FT.parse(path, params)
+            expect(data["title_id"]).to eq(row[0])
+            expect(data["unit_id"]).to eq(row[1])
+            expect(data["publication_date"]).to eq(row[2])
+            expect(data["rtype"]).to eq(row[3])
+            expect(data["mime"]).to eq(row[4])
+        end
+    end
+end