kishu 0.0.0

data/bin/kishu

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require File.expand_path("../../lib/kishu", __FILE__)
+
+Kishu::CLI.start

data/kishu.gemspec

@@ -0,0 +1,54 @@
+require "date"
+require File.expand_path("../lib/kishu/version", __FILE__)
+
+
+Gem::Specification.new do |spec|
+  spec.name          = "kishu"
+  spec.version       = Kishu::VERSION
+  spec.authors       = ["Kristian Garza"]
+  spec.email         = ["kgarza@datacite.org"]
+
+  spec.summary       = "Client for DOI Resolution Logs processing pipeline"
+  spec.description   = "This client helps you to prepare logs to be consumed for the pipeline as well as for creating DOI resolution reports using the COUNTER CoP "
+  spec.homepage      = "https://github.com/datacite/kishu"
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
+  # to allow pushing to a single host or delete this section to allow pushing to any host.
+  # if spec.respond_to?(:metadata)
+  #   spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
+  # else
+  #   raise "RubyGems 2.0 or newer is required to protect against " \
+  #     "public gem pushes."
+  # end
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+
+  # spec.add_dependency 'maremma', '>= 4.1', '< 5'
+  spec.add_dependency 'faraday', "~>0.15.3"
+  spec.add_dependency 'builder', '~> 3.2', '>= 3.2.2'
+  spec.add_dependency 'dotenv', '~> 2.1', '>= 2.1.1'
+  spec.add_dependency 'thor', '~> 0.19'
+  spec.add_dependency 'maremma', '>= 4.1', '< 5'
+  spec.add_dependency 'faraday_middleware-aws-sigv4', '~> 0.2.4'
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency 'elasticsearch', '~> 6.1.0'
+  spec.add_development_dependency "thor", '~> 0.19'
+  spec.add_development_dependency "faraday", "~>0.15.3"
+  spec.add_development_dependency 'rack-test', '~> 0'
+  spec.add_development_dependency 'vcr', '~> 3.0', '>= 3.0.3'
+  spec.add_development_dependency 'webmock', '~> 3.0', '>= 3.0.1'
+  spec.add_development_dependency 'simplecov', '~> 0.14.1'
+  spec.add_development_dependency 'factory_bot', '~> 4.0'
+  spec.add_dependency 'sucker_punch', '~> 2.0'
+  spec.add_dependency 'bolognese', '~> 0.9', '>= 0.10'
+  spec.add_dependency 'elasticsearch', '~> 6.1.0'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  spec.executables   = ["kishu"]
+  spec.require_paths = ["lib"]
+end

data/lib/kishu.rb

@@ -0,0 +1,30 @@
+require "kishu/resolution_event"
+require "kishu/usage_event"
+require "kishu/report"
+require "kishu/cli"
+require "kishu/sushi"
+require "kishu/version"
+require "kishu/client"
+require "kishu/log"
+require "kishu/pipeline"
+require "kishu/lagotto_job"
+
+
+API_URL         = ENV['API_URL'] ? ENV['API_URL'] : "https://api.datacite.org"
+HUB_URL         = ENV['HUB_URL'] ? ENV['HUB_URL'] : "https://api.test.datacite.org"
+HUB_TOKEN       = ENV['HUB_TOKEN'] ? ENV['HUB_TOKEN'] : ""
+ES_HOST         = ENV['ES_HOST'] ? ENV['ES_HOST'] : "localhost:9200"
+ES_INDEX        = ENV['ES_INDEX'] ? ENV['ES_INDEX'] : "resolutions"
+LOGSTASH_HOST   = ENV['LOGSTASH_HOST'] ? ENV['LOGSTASH_HOST'] : "localhost:9600"
+LAGOTTINO_URL   = ENV['LAGOTTINO_URL'] ? ENV['LAGOTTINO_URL'] : "https://api.test.datacite.org"
+LAGOTTINO_TOKEN = ENV['LAGOTTINO_TOKEN'] ? ENV['LAGOTTINO_TOKEN'] : ""
+LICENSE         = ENV['LICENSE'] ? ENV['LICENSE'] : "https://creativecommons.org/publicdomain/zero/1.0/"
+SOURCE_TOKEN    = ENV['SOURCE_TOKEN'] ? ENV['SOURCE_TOKEN'] : "65903a54-01c8-4a3f-9bf2-04ecc658247a"
+S3_MERGED_LOGS_BUCKET     = ENV['S3_MERGED_LOGS_BUCKET'] ? ENV['S3_MERGED_LOGS_BUCKET'] : "./monthly_logs"
+S3_RESOLUTION_LOGS_BUCKET = ENV['S3_RESOLUTION_LOGS_BUCKET'] ? ENV['S3_RESOLUTION_LOGS_BUCKET'] : "./"
+AWS_REGION                = ENV['AWS_REGION'] ? ENV['AWS_REGION'] : ""
+AWS_ACCESS_KEY_ID         = ENV['AWS_ACCESS_KEY_ID'] ? ENV['AWS_ACCESS_KEY_ID'] : ""
+AWS_SECRET_ACCESS_KEY     = ENV['AWS_SECRET_ACCESS_KEY'] ? ENV['AWS_SECRET_ACCESS_KEY'] : ""
+ELASTIC_PASSWORD          = ENV['ELASTIC_PASSWORD'] ? ENV['ELASTIC_PASSWORD'] : ""
+LOGS_TAG  = "[Resolution Logs]"
+puts ENV.to_a

data/lib/kishu/base.rb

@@ -0,0 +1,14 @@
+require 'elasticsearch'
+require 'json'
+require 'faraday'
+
+
+
+module Kishu
+  module Base
+    ES_HOST         = ENV['ES_HOST'] ? ENV['ES_HOST'] : "localhost:9200"
+
+    # __elasticsearch__ = Faraday.new(url: ES_HOST)
+    __elasticsearch__ = Elasticsearch::Client.new host: ES_HOST, transport_options: { request: { timeout: 3600, open_timeout: 3600 } }
+  end
+end

data/lib/kishu/cli.rb

@@ -0,0 +1,42 @@
+require 'thor'
+
+
+require_relative 'sushi'
+require_relative 'log'
+
+
+module Kishu
+  class CLI < Thor
+
+    include Kishu::Base
+    include Kishu::Utils
+    # include Kishu::Report
+    include Kishu::Merger
+    # include Kishu::Event
+
+    # load ENV variables from .env file if it exists
+    env_file = File.expand_path("../../.env", __FILE__)
+    if File.exist?(env_file)
+      require 'dotenv'
+      Dotenv.overload env_file
+    end
+
+    def self.exit_on_failure?
+      true
+    end
+
+    # from http://stackoverflow.com/questions/22809972/adding-a-version-option-to-a-ruby-thor-cli
+    map %w[--version -v] => :__print_version
+
+    desc "--version, -v", "print the version"
+    def __print_version
+      puts Kishu::VERSION
+    end
+    
+    desc "sushi SUBCOMMAND", "sushi commands"
+    subcommand "sushi", Kishu::Sushi
+
+    desc "log SUBCOMMAND", "log commands"
+    subcommand "log", Kishu::Log
+  end
+end

data/lib/kishu/client.rb

@@ -0,0 +1,89 @@
+require 'faraday_middleware'
+require 'faraday_middleware/aws_sigv4'
+require 'logger'
+
+require_relative 'utils'
+require_relative 'base'
+
+module Kishu
+  class Client 
+
+    def initialize
+
+      if ES_HOST == "localhost:9200" || ES_HOST == "elasticsearch:9200"
+        @client = Elasticsearch::Client.new(host: ES_HOST, user: "elastic", password: ELASTIC_PASSWORD, transport_options: { request: { timeout: 3600, open_timeout: 3600 }}) do |f|
+          f.adapter Faraday.default_adapter
+        end
+      else
+          @client = Elasticsearch::Client.new(host: ES_HOST, port: '80', scheme: 'http') do |f|
+            f.request :aws_sigv4,
+              service: 'es',
+              region: AWS_REGION,
+              access_key_id: AWS_ACCESS_KEY_ID,
+              secret_access_key: AWS_SECRET_ACCESS_KEY
+            f.adapter Faraday.default_adapter
+          end
+      end
+      @client
+    end
+
+
+    def get options={}
+
+      x =@client.search(body:{
+          size: options[:size] ||= 0,
+          query: {
+            query_string: {
+              query: "*"
+            }
+          },
+          aggregations: aggregations(options)
+        },
+        index: ES_INDEX
+        )
+      x
+    end
+
+    def is_empty?
+      return true unless get
+      nil
+    end
+
+    def clear_index
+      @client.indices.delete index: ES_INDEX
+      puts "Resolutions index has been deleted"
+    end
+
+
+    def get_logdate options={}
+      @client.search(body:{
+          size: 1,
+          query: {
+            query_string: {
+              query: "*"
+            }
+          },
+          aggregations: aggregations(options)
+        },
+        index: "resolutions"
+        ).dig("hits","hits",0,"_source","logdate")
+    end
+
+    def aggregations options={}
+      {
+        doi: {composite: {
+          sources: [{doi: {terms: {field: :doi	}}}],
+          after: { doi: options.fetch(:after_key,"")},
+          size: options[:aggs_size]
+          },
+          aggs: {
+            unique: {terms: {field: "unique_usage"}},
+            totale: {terms: {field: "total_usage"	}}
+          }
+        }
+      }
+    end
+
+  end
+end
+

data/lib/kishu/lagotto_job.rb

@@ -0,0 +1,22 @@
+require_relative 'resolution_event'
+
+
+class LagottoJob
+  include SuckerPunch::Job
+  include Kishu::Utils
+  workers 4 
+
+
+
+  def perform(report, options={})
+    # data = format_instance event, options
+
+    # push_url = LAGOTTINO_URL  + "/events"
+    # response =  Maremma.post(push_url, data: data.to_json,
+    #               bearer: LAGOTTINO_TOKEN,
+    #               content_type: 'application/vnd.api+json')
+    # puts data
+    # puts response.status
+    Report.send_report report
+  end
+end

data/lib/kishu/log.rb

@@ -0,0 +1,33 @@
+
+require 'thor'
+
+require_relative 'merger'
+require_relative 'utils'
+require_relative 'base'
+
+module Kishu
+  class Log < Thor
+
+  include Kishu::Base
+  include Kishu::Merger
+  include Kishu::Utils
+
+
+   desc "create logs", "create logs"
+   method_option :logs_bucket,   :default => ENV['S3_RESOLUTION_LOGS_BUCKET']
+   method_option :output_bucket, :default => ENV['S3_MERGED_LOGS_BUCKET']
+   method_option :month_year,    :type => :string, :default => "201804"
+
+   def create
+      return "Logs don't exist" unless File.directory?(options[:month_year])
+      return "Pipeline has events" unless Pipeline.new.is_empty?
+      @log_date = get_date options[:month_year]
+      @folder   = options[:month_year]  
+      puts @log_date
+      uncompress_files
+      # add_bookends
+      merge_files
+      sort_files
+   end
+  end
+end

data/lib/kishu/merger.rb

@@ -0,0 +1,69 @@
+
+require 'date'
+
+module Kishu
+  module Merger
+
+    FILE_STEM = "DataCite-access.log"
+
+    def get_date filename
+      Date.parse("#{filename}01")
+    end
+
+
+    def uncompress_files
+      system("gunzip #{resolution_logs_folder}/#{FILE_STEM}-*")
+    end
+
+
+    # def add_bookends
+    #   File.delete("#{resolution_logs_folder}/#{FILE_STEM}-1-begin.log") if File.exist?("#{resolution_logs_folder}/#{FILE_STEM}-1-begin.log")
+    #   File.delete("#{resolution_logs_folder}/#{FILE_STEM}-9-eof.log") if File.exist?("#{resolution_logs_folder}/#{FILE_STEM}-9-eof.log")
+
+    #   begin_date = Date.civil(@log_date.year,@log_date.month,1).strftime("%Y-%m-%d")
+    #   end_date   = Date.civil(@log_date.year,@log_date.month+1, 1).strftime("%Y-%m-%d") 
+
+    #   begin_line = '0.0.0.0 HTTP:HDL "'+begin_date+' 00:00:00.000Z" 1 1 22ms 10.5281/zenodo.1043571 "300:10.admin/codata" "" "Mozilla"'+"\n"
+    #   puts begin_line
+
+    #   end_line = '0.0.0.0 HTTP:HDL "'+end_date+' 00:01:00.000Z" 1 1 22ms 10.5281/zenodo.1043571 "300:10.admin/codata" "" "Mozilla"'+"\n"
+    #   puts end_line
+
+    #   File.open("#{resolution_logs_folder}/#{FILE_STEM}-1-begin.log","w") {|f| f.write(begin_line) }
+    #   File.open("#{resolution_logs_folder}/#{FILE_STEM}-9-eof.log","w") {|f| f.write(end_line) }
+    # end
+
+    def merged_file
+      "#{merged_logs_folder}/datacite_resolution_logs_#{@log_date}.log"
+    end
+
+    def sorted_file
+      "#{resolution_logs_folder}/datacite_resolution_logs_#{@log_date}_sorted.log"
+    end
+
+    def resolution_logs_folder
+      bucket = S3_RESOLUTION_LOGS_BUCKET
+      "#{bucket}#{@folder}"
+    end
+
+    def merged_logs_folder
+      bucket = S3_MERGED_LOGS_BUCKET
+      "#{bucket}#{@folder}"
+    end
+
+    def merge_files
+      File.delete(merged_file) if File.exist?(merged_file)
+
+      system("cat #{resolution_logs_folder}/#{FILE_STEM}-* > #{merged_file}")
+      puts "Merged Completed"
+    end
+
+    def sort_files
+      File.delete(sorted_file) if File.exist?(sorted_file)
+
+      system("sort -k3 #{merged_file} > #{sorted_file}")
+      puts "Sorted Completed"
+      puts sorted_file
+    end
+  end
+end

data/lib/kishu/pipeline.rb

@@ -0,0 +1,29 @@
+require 'faraday'
+require 'logger'
+
+require_relative 'utils'
+require_relative 'base'
+
+module Kishu
+  class Pipeline 
+
+    def initialize
+      @conn = Faraday.new(:url => LOGSTASH_HOST)
+      # logger = Logger.new(STDOUT)
+      # logger.info 
+    end
+
+    def is_ready?
+      main  = @conn.get do |req|
+        req.url '/_node/stats/pipelines/main'
+      end
+      return nil unless main.dig("pipelines","main","events","out") == 0
+    end
+
+    def is_running?
+
+    end
+
+  end
+end
+

data/lib/kishu/report.rb

@@ -0,0 +1,149 @@
+require 'json'
+require 'date'
+require "faraday"
+require 'securerandom'
+require 'zlib'
+require 'digest'
+
+require_relative 'resolution_event'
+require_relative 'client'
+
+module Kishu
+  class Report
+
+    include Kishu::Base
+    include Kishu::Utils
+
+    def initialize options={}
+      set_period 
+      @es_client = Client.new()
+      @logger = Logger.new(STDOUT)
+      @report_id = options[:report_id] ? options[:report_id] : ""
+      @total = 0
+      @aggs_size = options[:aggs_size] 
+      @chunk_size = options[:chunk_size]
+      @after = options[:after_key] ||=""
+    end
+
+    def report_period options={}
+      es_client = Client.new()
+
+      logdate = es_client.get_logdate({aggs_size: 1})
+      puts logdate
+      Date.parse(logdate)
+    end
+
+
+    def get_events options={}
+      logger = Logger.new(STDOUT)
+      es_client = Client.new()
+      response = es_client.get({aggs_size: @aggs_size || 500, after_key: options[:after_key] ||=""})
+      aggs = response.dig("aggregations","doi","buckets")
+      x = aggs.map do |agg|
+        ResolutionEvent.new(agg,{period: @period, report_id: @report_id}).wrap_event
+      end
+      after = response.dig("aggregations","doi").fetch("after_key",{"doi"=>nil}).dig("doi")
+      logger.info "After_key for pagination #{after}"
+      y = {data: x, after: after}
+      y
+    end
+
+
+    def generate_dataset_array
+      @datasets = []
+      loop do
+        response = get_events({after_key: @after ||=""})
+        @datasets = @datasets.concat response[:data]
+        @after = response[:after]
+        @total += @datasets.size
+        generate_chunk_report if @datasets.size > @chunk_size
+        break if @after.nil?
+      end
+    end
+
+    def compress report
+      # report = File.read(hash)
+      gzip = Zlib::GzipWriter.new(StringIO.new)
+      string = report.to_json
+      gzip << string
+      body = gzip.close.string
+      body
+    end
+
+ 
+    def generate_chunk_report
+      # puts get_template
+      # LagottoJob.perform_async(get_template(@datasets))
+      file = merged_file #+ 'after_key_' + @after
+      File.open(file,"w") do |f|
+        f.write(JSON.pretty_generate get_template)
+      end
+      send_report get_template
+      @datasets = []
+    end
+
+    def make_report options={}
+      generate_dataset_array
+      @logger.info  "#{LOGS_TAG} Month of #{@period.dig("begin-date")} sent to Hub in report #{@uid} with stats for #{@total} datasets"
+    end
+
+
+    def set_period 
+      report_period
+      @period = { 
+        "begin-date": Date.civil(report_period.year, report_period.mon, 1).strftime("%Y-%m-%d"),
+        "end-date": Date.civil(report_period.year, report_period.mon, -1).strftime("%Y-%m-%d"),
+      }
+    end
+
+    def send_report report, options={}
+      uri = HUB_URL+'/reports'   
+      puts uri
+
+      headers = {
+        content_type: "application/gzip",
+        content_encoding: 'gzip',
+        accept: 'gzip'
+      }
+      
+      body = compress(report)
+      n = 0
+      loop do
+        request = Maremma.post(uri, data: body,
+          bearer: ENV['HUB_TOKEN'],
+          headers: headers,
+          timeout: 100)
+
+        @uid = request.body.dig("data","report","id")
+        @logger.info "#{LOGS_TAG} Hub response #{request.status} for Report finishing in #{@after}"
+        @logger.info "#{LOGS_TAG} Hub response #{@uid} for Report finishing in #{@after}"
+        n += 1
+        break if request.status == 201
+        fail "#{LOGS_TAG} Too many attempts were tried to push this report" if n > 1
+        sleep 1
+      end
+    end
+
+    def get_template 
+      {
+      "report-header": get_header,
+      "report-datasets": @datasets
+      }
+    end
+
+    def get_header 
+      {
+        "report-name": "resolution report",
+        "report-id": "dsr",
+        release: "drl",
+        created: Date.today.strftime("%Y-%m-%d"),
+        "created-by": "datacite",
+        "reporting-period": @period,
+        "report-filters": [],
+        "report-attributes": [],
+        exceptions: [{code: 69,severity: "warning", message: "Report is compressed using gzip","help-url": "https://github.com/datacite/sashimi",data: "usage data needs to be uncompressed"}]
+      }
+    end
+
+  end
+end