logstash-filter-ezproxy 0.1.0

data/lib/logstash/filters/jstor.rb

@@ -0,0 +1,112 @@
+require 'uri'
+require 'cgi'
+
+module Jstor
+    def Jstor.parse (input)
+
+        uri = URI(URI.unescape(input))
+
+        url = uri.path
+        params = {}
+        if (uri.query)
+            params = CGI::parse(uri.query)
+        end
+
+        data = {
+            "provider" => "jstor"
+        }
+        doi_prefix = "10.2307"
+
+
+        if (match = /^\/journal\/([a-z0-9]+)$/i.match(url))
+            data["rtype"] = "TOC"
+            data["mime"] = "MISC"
+            data["unit_id"] = match[1]
+            data["title_id"] = match[1]
+
+        elsif (match = /^\/stable\/10\.[0-9]+\/(([a-z]+)\.([0-9]+)\.([0-9]+)\.issue-([0-9]+))$/i.match(url))
+            data["rtype"] = "TOC"
+            data["mime"] = "MISC"
+            data["unit_id"] = match[1]
+            data["title_id"] = match[2]
+            data["issue"] = match[5]
+
+            if match[3].length >= 4
+                data["publication_date"] = match[3]
+                data["vol"] = match[4]
+            else
+                data["vol"] = match[3]
+            end
+
+        elsif (match = /^\/stable\/((10\.[0-9]+\/)?([a-z0-9]+))$/i.match(url))
+            data["rtype"] = "TOC"
+            data["mime"] = "MISC"
+            data["unit_id"] = match[3]
+            data["title_id"] = match[3]
+
+            if match[2]
+                data["doi"] = match[1]
+            end
+
+        elsif (match =  /^\/stable\/(i[0-9]+)$/i.match(url))
+            data["rtype"] = "TOC"
+            data["mime"] = "MISC"
+            data["unit_id"] = match[1]
+            data["title_id"] = match[1]
+
+        elsif (/^\/action\/showPublication$/i.match(url))
+            if (params["journalCode"])
+                data["title_id"] = params["journalCode"][0]
+                data["unit_id"] = params["journalCode"][0]
+                data["rtype"] = 'TOC'
+                data["mime"] = 'MISC'
+            end
+
+        elsif (match =  /^\/stable\/(get_image|pdf|pdfplus)\/((10\.[0-9]+\/)?([a-z0-9.]+?))(?:\.pdf)?$/i.match(url))
+            data["unit_id"] = match[4]
+            data["doi"] = match[3] ? match[2] : doi_prefix + "/" + match[2] 
+            
+            case match[1]
+            when 'get_image'
+                data["rtype"] = "ARTICLE_SECTION"
+                data["mime"] = "GIF"
+            when 'pdf'
+                data["rtype"] = "ARTICLE"
+                data["mime"] = "PDF"
+            when 'pdfplus'
+                data["rtype"] = "ARTICLE"
+                data["mime"] = "PDFPLUS"
+            end
+
+            idPattern = /^([a-z0-9]+)((?:\.(\d+))?\.(\d+)\.(\d+)\.(\w+))?/.match(match[4]) || [];
+
+            data["title_id"] = idPattern[1]
+            data["publication_date"] = idPattern[3]
+            data["vol"] = idPattern[4]
+            data["issue"] = idPattern[5]
+
+            if (idPattern[6] == 'cover')
+                data["rtype"] = 'COVER'
+            elsif (idPattern[6] == 'toc')
+                data["rtype"] = 'TOC'
+            else
+                if (idPattern[6] != nil)
+                    first_page = idPattern[6].to_i
+                    
+                    unless (first_page.to_f.nan?)
+                        data["first_page"] = first_page.to_s
+                    end
+                end
+            end
+
+        elsif (match = /^\/stable\/(info|view)\/([0-9]+)$/i.match(url))
+            data["rtype"] = match[1] === 'info' ? "ABS" : "PREVIEW"
+            data["mime"] = "MISC"
+            data["unit_id"] = match[2]
+            data["title_id"] = match[2]
+            data["issue"] = match[5]
+        end
+
+        return data
+    end
+end

data/lib/logstash/filters/lexisnexis.rb

@@ -0,0 +1,37 @@
+require 'uri'
+
+module LexisNexis
+  def LexisNexis.parse (input)
+    uri = URI(URI.unescape(input))
+
+    path = uri.path
+
+    data = {
+      "provider" => "lexisnexis"
+    }
+
+    if (/\/droit\/results\/docview\/docview/.match(path))
+      if (uri.query)
+        params = CGI::parse(uri.query)
+
+        if (params["risb"])
+          data["title_id"] = params["risb"][0]
+          data["unit_id"] = params["risb"][0]
+        end
+
+        if (params["format"])
+          case params["format"][0]
+            when 'GNBFULL'
+              data["rtype"] = 'ARTICLE'
+              data["mime"] = 'HTML'
+            when 'AUTRECAS' 
+              data["rtype"] = 'ARTICLE'
+              data["mime"] = 'HTML'
+          end
+        end
+      end
+    end
+    
+    return data
+  end
+end

data/lib/logstash/filters/sage.rb

@@ -0,0 +1,39 @@
+require "uri"
+
+module Sage
+  def Sage.parse (input)
+    uri = URI(URI.unescape(input))
+
+    path = uri.path
+
+    data = {
+      "provider" => "sage"
+    }
+
+    if (match = /^\/(loi|toc)\/(([a-z]+)\/?([0-9]+)?\/?([0-9]+)?)$/i.match(path))
+      data["rtype"] = "TOC"
+      data["mime"] = "MISC"
+      data["title_id"] = match[3]
+      if (match[4])
+        data["unit_id"] = match[2]
+      end
+
+    elsif (match = /^\/doi\/([a-z]+)\/(([0-9]{2})\.([0-9]{4})\/([0-9]+))$/i.match(path))
+      data["rtype"] = "ARTICLE"
+      data["mime"] = "HTML"
+      data["doi"] = match[2]
+      data["unit_id"] = match[5]
+
+      case (match[1])
+      when "pdf"
+        data["mime"] = "PDF"
+      when "figure"
+        data["rtype"] = "FIGURE"
+      end
+    end
+
+    return data
+
+  end
+
+end

data/lib/logstash/filters/sciencedirect.rb

@@ -0,0 +1,171 @@
+require 'uri'
+
+module ScienceDirect
+  def ScienceDirect.parse (input)
+    uri = URI(URI.unescape(input))
+
+    path = uri.path
+    params = {}
+
+    if (uri.query)
+      params = CGI::parse(uri.query)
+    end
+
+    data = {
+      "provider" => "sciencedirect"
+    }
+
+    if (params.key?("_ob"))
+      if (params['_cdi'])
+        data['title_id'] = params['_cdi'][0]
+      end
+
+      case (params["_ob"][0])
+      when 'PdfDownloadURL'
+        data['mime'] = 'PDF'
+
+        data['rtype'] = 'ARTICLES_BUNDLE'
+
+        data['unit_id'] = params['_hubEid'][0]
+        data["pii"]    = (params['_hubEid'][0] || '').split('-')[2]
+
+
+        if (params.key?("_isbn")|| params.key?('isBook'))
+          data['rtype']            = 'CHAPTERS_BUNDLE'
+          data['print_identifier'] = params['_isbn'][0]
+          data['title_id']         = params['_isbn'][0]
+          data['unit_id']          = params['_isbn'][0]
+        end
+
+        if (data["pii"])
+          data['title_id'] = data["pii"][1, 8]
+          data['print_identifier'] = data["pii"][1, 4] + "-" + data["pii"][5, 4]
+        end
+
+      when 'IssueURL'
+        data['title_id'] = (params['_tockey'][0] || '').split('#')[2]
+        data['rtype']    = 'TOC'
+        data['mime']     = 'MISC'
+
+      when 'ArticleURL'
+        case (params['_fmt'][0])
+        when 'summary'
+          data['rtype'] = 'ABS'
+          data['mime']  = 'MISC'
+        when 'full'
+          data['rtype'] = 'ARTICLE'
+          data['mime']  = 'HTML'
+        end
+
+      when 'MImg'
+        data['rtype'] = 'ARTICLE'
+        data['mime']  = 'PDF'
+      
+      when 'MiamiImageURL'
+        if (params['_pii'])
+
+          data["pii"]      = params['_pii'][0]
+          data['unit_id']   = params['_pii'][0]
+
+          if (params['_pii'][0][0] === 'B')
+            data['print_identifier'] = params['_pii'][0][1, 13]
+            data['title_id']         = data['print_identifier']
+            data['rtype']            = 'BOOK_SECTION'
+            data['mime']             = 'PDF'
+
+          else
+            data['print_identifier'] = params['_pii'][0][1, 4] + "-" + params['_pii'][0][5, 4]
+            data['title_id']         = params['_pii'][0][1, 8]
+            data['rtype']            = 'ARTICLE'
+            data['mime']             = 'PDF'
+          end
+        end
+
+      when 'PdfExcerptURL'
+        data['rtype'] = 'PREVIEW'
+        data['mime']  = 'PDF'
+
+        if (params['_imagekey'][0] && params['_piikey'][0])
+          data["pii"] = params['_piikey'][0]
+          if (match = /.?-[^-]+-([0-9]{4})([0-9]{3}[0-9Xx])([0-9A-Za-z]*)-main.pdf$/.match(params['_imagekey'][0]))
+            data['unit_id']          = params['_piikey'][0]
+            data['title_id']         = match[1] + match[2]
+            data['print_identifier'] = match[1] + '-' + match[2]
+          end
+        end
+      end
+    
+    elsif (match = /^\/science\/article\/pii\/(([SB])?([0-9]{7}(?:[0-9]{5})?[0-9Xx])[0-9A-Za-z]*)(\/pdf(?:ft)?)?$/.match(path))
+                  
+      data["pii"]    = match[1]
+      data['unit_id'] = match[1]
+      data['mime']   = match[4] ? 'PDF' : 'HTML'
+
+      if (match[2] == 'B')
+        data['rtype']            = 'BOOK_SECTION'
+        data['title_id']         = match[3]
+        data['print_identifier'] = match[3]
+      else
+        data['rtype']            = 'ARTICLE'
+        data['title_id']         = match[3][0, 8]
+        data['print_identifier'] = match[3][0, 4] + "-" + match[3][4, 4]
+      end
+    
+
+    elsif ((match = /^\/science\/(journal|bookseries|handbooks|handbooks|book)\/([0-9Xx]{8,})(\/[0-9]+)?(\/[0-9]+)?$/.match(path)))
+                          
+      data['rtype']    = 'TOC'
+      data['mime']     = 'MISC'
+      data['unit_id']   = match[2]
+      data['title_id'] = match[2]
+
+      if (match[3])
+        data['unit_id'] += match[3]
+      end
+
+      case (match[1])
+      when 'journal', 'handbooks', 'bookseries'
+        data['print_identifier'] = match[2][0, 4] + "-" + match[2][4, 4]
+      when 'book'
+        data['print_identifier'] = match[2]
+      end
+
+    elsif (match = /^\/science\/MiamiMultiMediaURL\/[^\/]+(S([0-9]{4})([0-9]{3}[0-9Xx])[a-zA-Z0-9]*).*\.pdf$/.match(path))
+    
+      data["pii"]              = match[1]
+      data['unit_id']          = match[1]
+      data['title_id']         = match[2] + match[3]
+      data['print_identifier'] = match[2] + "-" + match[3]
+      data['rtype']            = 'ARTICLE'
+      data['mime']             = 'PDF'
+
+    elsif (match = /^\/(([SB])?([0-9]{7}(?:[0-9]{5})?[0-9Xx])[0-9A-Za-z]*)\/[0-9A-Za-z\-.]*-main\.pdf$/.match(path))
+
+      data["pii"]    = match[1]
+      data['unit_id'] = match[1]
+      data['mime']   = 'PDF'
+
+      if (match[2] === 'B')
+        data['rtype']            = 'BOOK_SECTION'
+        data['title_id']         = match[3]
+        data['print_identifier'] = match[3]
+      else
+        data['rtype']            = 'ARTICLE'
+        data['title_id']         = match[3][0, 8]
+        data['print_identifier'] = match[3][0, 4] + "-" + match[3][4, 4]
+      end
+
+    elsif (path == '/science/publication')
+      data['rtype'] = 'TOC'
+      data['mime']  = 'MISC'
+
+      if (params['issn'])
+        data['print_identifier'] = params['issn'][0][0, 4] + "-" + params['issn'][0][4, 4]
+        data['unit_id']          = data['print_identifier']
+        data['title_id']         = params['issn'][0]
+      end
+    end
+
+    return data
+  end
+end

data/lib/logstash/filters/tandf.rb

@@ -0,0 +1,55 @@
+
+require 'uri'
+require 'cgi'
+
+module TandF
+  def TandF.parse (input)
+
+    uri = URI(URI.unescape(input))
+
+    path = uri.path
+    params = {}
+    if (uri.query)
+        params = CGI::parse(uri.query)
+    end
+
+    data = {
+        "provider" => "tandf"
+    }
+
+    if ((match = /^\/doi\/(full|pdf|abs)\/([0-9.]+\/([0-9a-z.]+))$/.match(path)))
+      data['doi']    = match[2]
+      data['unit_id'] = match[3]
+
+      if (/^[0-9]{8}/.match(match[3]))
+        data['print_identifier'] = match[3][0, 4] + '-' + match[3][4, 4]
+        data['title_id']         = data['print_identifier']
+      end
+
+      if (match[1].upcase == 'FULL')
+        data['rtype'] = 'ARTICLE'
+        data['mime']  = 'HTML'
+      elsif (match[1].upcase == 'PDF')
+        data['rtype'] = 'ARTICLE'
+        data['mime']  = 'PDF'
+      elsif (match[1].upcase == 'ABS')
+        data['rtype'] = 'ABS'
+        data['mime']  = 'HTML'
+      end
+    
+    elsif ((match = /^\/toc\/([a-zA-Z0-9]+)\/current$/.match(path)))
+      data['rtype']    = 'TOC'
+      data['mime']     = 'HTML'
+      data['title_id'] = match[1]
+      data['unit_id']  = match[1]
+    elsif ((match = /^\/loi\/([a-zA-Z0-9]+)$/.match(path)))
+      data['rtype']    = 'TOC'
+      data['mime']     = 'HTML'
+      data['title_id'] = match[1]
+      data['unit_id']  = match[1]
+    end
+
+
+    return data;
+  end
+end

data/lib/logstash/filters/wiley.rb

@@ -0,0 +1,202 @@
+require 'uri'
+
+module Wiley
+  def Wiley.parse (input)
+    uri = URI(URI.unescape(input))
+
+    path = uri.path
+
+    data = {
+      "provider" => "wiley"
+    }
+
+    #!/usr/bin/env node
+        
+    if ((match = /\/journal\/(10\.[0-9]+\/(\(ISSN\)([0-9]{4}-[0-9]{3}[0-9xX])))/i.match(path)))
+      data['doi']    = match[1];
+      data['unit_id'] = match[2];
+      data['rtype']  = 'TOC';
+      data['mime']   = 'MISC';
+
+      data['online_identifier'] = match[3];
+
+    elsif ((match = /^\/doi\/(10\.[0-9]+\/(([^.]+)\.([0-9]{4})\.[^.]+\.[^.]+))\/issuetoc$/i.match(path)))
+      data['doi']      = match[1];
+      data['unit_id']   = match[2];
+      data['title_id'] = match[3].upcase;
+      data['rtype']    = 'TOC';
+      data['mime']     = 'MISC';
+
+    data['publication_date'] = match[4];
+
+    elsif ((match = /^\/doi\/(10\.[0-9]+\/(j\.([0-9]{4}-[0-9]{3}[0-9xX])\.([0-9]{4})\.[^.]+\.[^.]+))\/abstract$/i.match(path)))
+      data['doi']    = match[1];
+      data['unit_id'] = match[2];
+      data['rtype']  = 'ABS';
+      data['mime']   = 'MISC';
+
+    data['online_identifier'] = match[3];
+    data['publication_date']  = match[4];
+
+    elsif ((match = /^\/doi\/(10\.[0-9]+\/(([^.]+)\.([0-9]{4})[0-9]+))\/abstract$/i.match(path)))
+      data['doi']      = match[1];
+      data['unit_id']   = match[2];
+      data['title_id'] = match[3].upcase;
+      data['rtype']    = 'ABS';
+      data['mime']     = 'MISC';
+
+    data['publication_date'] = match[4];
+
+    elsif ((match = /^\/doi\/(10\.[0-9]+\/(([^.]+)\.[0-9]+))\/full$/i.match(path)))
+      data['doi']      = match[1];
+      data['unit_id']   = match[2];
+      data['title_id'] = match[3].upcase;
+      data['rtype']    = 'ARTICLE';
+      data['mime']     = 'HTML';
+
+    elsif ((match = /^\/doi\/(10\.[0-9]+\/(j\.([0-9]{4}-[0-9]{3}[0-9xX])\.([0-9]{4})\.[^.]+\.[^.]+))\/pdf$/i.match(path)))
+      data['doi']    = match[1];
+      data['unit_id'] = match[2];
+      data['rtype']  = 'ARTICLE';
+      data['mime']   = 'PDF';
+
+      data['online_identifier'] = match[3];
+      data['publication_date']  = match[4];
+
+    elsif ((match = /^\/doi\/(10\.[0-9]+\/(([^.]+)\.[0-9]+))\/pdf$/i.match(path)))
+      data['doi']      = match[1];
+      data['unit_id']   = match[2];
+      data['title_id'] = match[3].upcase;
+      data['rtype']    = 'ARTICLE';
+      data['mime']     = 'PDF';
+
+    elsif ((match = /^\/book\/(10\.[0-9]+\/([0-9]+))$/i.match(path)))
+      data['doi']      = match[1];
+      data['unit_id']   = match[2];
+      data['title_id'] = match[2].upcase;
+      data['rtype']    = 'TOC';
+      data['mime']     = 'MISC';
+
+      data['print_identifier']  = match[2];
+
+    elsif (match = /^\/doi\/(10\.[0-9]+\/(([0-9]+)\.[^.]+))\/pdf$/i.match(path))
+      data['doi']      = match[1];
+      data['unit_id']   = match[2];
+      data['title_id'] = match[3].upcase;
+      data['rtype']    = 'BOOK_SECTION';
+      data['mime']     = 'PDF';
+
+      data['print_identifier']  = match[3];
+
+    elsif (match = /^\/enhanced\/doi\/(10\.[0-9]+\/(([^.]+)\.[^\/]+))\/?$/i.match(path))
+      data['doi']      = match[1];
+      data['unit_id']   = match[2];
+      data['title_id'] = match[3].upcase;
+      data['rtype']    = 'ARTICLE';
+      data['mime']     = 'HTML';
+
+    elsif ((match = /^\/enhanced\/doi\/(10\.[0-9]+\/(([0-9]{4})([a-z0-9]{2})[a-z0-9]+))\/?$/i.match(path)))
+
+      data['doi']      = match[1];
+      data['unit_id']   = match[2];
+      data['title_id'] = match[4].upcase;
+      data['rtype']    = 'ARTICLE';
+      data['mime']     = 'HTML';
+
+      data['publication_date'] = match[3];
+
+    elsif ((match = /^\/agu\/issue\/(10\.[0-9]+\/(([^.]+)\.[^\/]+))\/?$/i.match(path)))
+      data['doi']      = match[1];
+      data['unit_id']   = match[2];
+      data['title_id'] = match[3].upcase;
+      data['rtype']    = 'TOC';
+      data['mime']     = 'HTML';
+
+    elsif (/^\/readcube$/i.match(path))
+      data['rtype'] = 'ARTICLE';
+      data['mime']  = 'READCUBE';
+
+      if (uri.query)
+        params = CGI::parse(uri.query)
+
+        if (params["resource"])
+
+          data['doi']    = params["resource"][0];
+          data['unit_id'] = params["resource"][0].split('/')[1];
+          if ((match = /(10\.[0-9]+)\/([0-9]{4})([a-z0-9]{2})([^\/]+)$/i.match(params['resource'][0])))
+              data['title_id'] = match[3].upcase;
+          end
+        end
+      end
+
+
+    elsif ((match = /^\/doi\/(10\.[0-9]+\/([a-z]{1}[0-9]{8}([0-9]{2})[a-z0-9]+))\/pdf$/i.match(path)))
+      data['doi']      = match[1];
+      data['unit_id']   = match[2];
+      data['title_id'] = match[2].upcase;
+      data['rtype']    = 'ARTICLE';
+      data['mime']     = 'PDF';
+
+      data['publication_date'] = '20' + match[3];
+
+    elsif ((match = /^\/iucr\/(10\.[0-9]+\/([a-z]{1}[0-9]{8}([0-9]{2})[0-9a-z]+))/i.match(path)))
+      data['doi']      = match[1];
+      data['unit_id']   = match[2] ;
+      data['title_id'] = match[2].upcase;
+      data['rtype']    = 'ARTICLE';
+      data['mime']     = 'HTML';
+
+      data['publication_date'] = '20' + match[3];
+
+    elsif ((match = /^\/doi\/(10\.[0-9]+\/([a-z]{1}[0-9]{8}([0-9]{2})[0-9a-z]+))\/([a-z]+)$/i.match(path)))
+      data['doi']      = match[1];
+      data['unit_id']   = match[2];
+      data['title_id'] = match[2].upcase;
+
+      data['publication_date'] = '20' + match[3];
+
+      case (match[4])
+      when 'abstract'
+        data['rtype'] = 'ABS';
+        data['mime']  = 'MISC';
+      when 'pdf'
+        data['rtype'] = 'ARTICLE';
+        data['mime']  = 'PDF';
+      when 'full'
+        data['mime']  = 'PDF';
+        data['rtype'] = 'HTML';
+      end
+
+    elsif ((match = /^\/store\/(10\.[0-9]+\/(([a-z]+)\.([0-9]{4})[0-9]+))\/asset\/[a-z]+[0-9]+.pdf$/i.match(path)))
+      data['doi']      = match[1];
+      data['unit_id']   = match[2];
+      data['title_id'] = match[3].upcase;
+      data['mime']     = 'PDF';
+
+      data['publication_date'] = match[4];
+
+    elsif ((match = /^\/doi\/(10\.[0-9]+\/(([0-9]{2,4})([a-z]+)[0-9]+))\/(pdf|full)$/i.match(path)))
+      data['doi']      = match[1];
+      data['unit_id']   = match[2];
+      data['title_id'] = match[4].upcase;
+      data['rtype']    = 'ARTICLE';
+      data['mime']     = match[5] == 'pdf' ? 'PDF' : 'HTML';
+
+      data['publication_date'] = match[3];
+
+      if (match[3].length === 2)
+        data['publication_date'] = '19' + match[3];
+      end
+
+    elsif ((match = /^\/doi\/(10\.[0-9]+\/([^.]+))\/(pdf|full)$/i.match(path)))
+      data['doi']    = match[1];
+      data['unit_id'] = match[2];
+      data['rtype']  = 'ARTICLE';
+      data['mime']   = match[3] === 'pdf' ? 'PDF' : 'HTML';
+    end
+
+    return data
+
+  end
+
+end