kishu 0.0.0

data/lib/kishu/resolution_event.rb

@@ -0,0 +1,83 @@
+require 'faraday'
+require 'logger'
+require 'maremma'
+require 'sucker_punch'
+
+require_relative 'utils'
+require_relative 'base'
+require_relative 'lagotto_job'
+
+module Kishu
+  class ResolutionEvent 
+
+    include Kishu::Utils
+
+    def initialize(event, options={})
+      @event = event
+      @logger = Logger.new(STDOUT)
+      @period = options[:period]
+    end
+    
+    def wrap_event
+      # puts "------------------ \n"
+      totale = @event.dig("totale").fetch("buckets", [])
+      # puts @event.dig("unique").fetch("buckets", nil)
+      unique = @event.dig("unique").fetch("buckets", [])
+      # puts unique[1].dig('key')
+    
+      unique_regular = unique.find_all {|access_method| access_method.fetch('key',"").match('regular') }
+      unique_machine = unique.find_all {|access_method| access_method.fetch('key',"").match('machine') }
+      total_regular  = totale.find_all {|access_method| access_method.fetch('key',"").match('regular') }
+      total_machine  = totale.find_all {|access_method| access_method.fetch('key',"").match('machine') }
+
+      dataset = { 
+        doi: @event.dig("key","doi"), 
+        unique_counts_regular: unique_regular.empty? ? 0 : unique_regular.size,
+        unique_counts_machine: unique_machine.empty? ? 0 : unique_machine.size,
+        total_counts_regular: total_regular.empty? ? 0 : total_regular.dig(0,"doc_count"),
+        total_counts_machine: total_machine.empty? ? 0 :  total_machine.dig(0,"doc_count")
+      }
+
+    
+        @doi = dataset.fetch(:doi,nil)
+
+        data = {}
+        instances =[
+          {
+            "count" => dataset.fetch(:total_counts_regular),
+            "access-method" => "regular",
+            "metric-type" => "total-resolutions"
+          },
+          {
+            "count" => dataset.fetch(:unique_counts_regular),
+            "access-method" => "regular",
+            "metric-type" => "unique-resolutions"
+          },
+          {
+            "count" => dataset.fetch(:unique_counts_machine),
+            "access-method" => "machine",
+            "metric-type" => "unique-resolutions"
+          },
+          {
+            "count" => dataset.fetch(:total_counts_machine),
+            "access-method" => "machine",
+            "metric-type" => "total-resolutions"
+          },
+        ]
+    
+        instances.delete_if {|instance| instance.dig("count") < 1}
+  
+
+        instanced = { 
+          "dataset-id" => [{"type" => "doi", "value"=> dataset.fetch(:doi,nil)}],
+          "performance" => [{
+            "period"=> @period,
+            "instance"=> instances
+          }]
+        }
+      instanced
+    end
+    
+  
+  end
+end

data/lib/kishu/s3.rb

@@ -0,0 +1,24 @@
+
+require 'aws-sdk-s3'
+
+require_relative 'utils'
+require_relative 'base'
+
+module Kishu
+  class S3 
+
+    def initialize
+      s3 = Aws::S3::Client.new
+      resp = s3.list_buckets
+      resp.buckets.map(&:name)
+    end
+
+    def download_logs
+      resp = s3.get_object(
+        response_target: '/logs',
+        bucket: S3_RESOLUTION_LOGS_BUCKET,
+        key: 'object-key')
+      resp.metadata
+    end
+  end
+end

data/lib/kishu/sushi.rb

@@ -0,0 +1,59 @@
+
+require 'thor'
+
+
+require_relative 'resolution_event'
+require_relative 'report'
+require_relative 'utils'
+require_relative 'base'
+
+module Kishu
+  class Sushi < Thor
+
+  include Kishu::Base
+  include Kishu::Utils
+
+
+   desc "get sushi", "get resolution report"
+  #  method_option :username, :default => ENV['MDS_USERNAME']
+   method_option :aggs_size, :type => :numeric, :default => 1000
+   method_option :month_year, :type => :string, :default => "2018-04"
+   def get
+    x =Report.new()
+    x.make_report(options)
+    
+   end
+
+   method_option :month_year, :type => :string, :default => "2018-04"
+   method_option :after_key, :type => :string
+   def continue_report
+    x =Report.new()
+    x.generate_files(options)
+    
+   end
+
+   desc "clean_all sushi", "clean index"
+   method_option :month_year, :type => :string, :default => "2018-04"
+   method_option :after_key, :type => :string
+   def clean_all
+    x =Client.new()
+    x.clear_index
+    
+   end
+
+
+   desc "send_report_events sushi", "send_report_events index"
+   method_option :month_year, :type => :string, :default => "2018-04"
+   method_option :after_key, :type => :string
+   method_option :chunk_size, :type => :numeric, :default => 40000
+   method_option :aggs_size, :type => :numeric, :default => 500
+   def send_report_events
+    fail "You need to set your JWT" if HUB_TOKEN.blank?
+    x =Report.new(options)
+    x.make_report(options)
+    
+   end
+
+   
+  end
+end

data/lib/kishu/usage_event.rb

@@ -0,0 +1,124 @@
+require 'faraday'
+require 'logger'
+require 'maremma'
+
+require_relative 'utils'
+require_relative 'base'
+
+module Kishu
+  class UsageEvent 
+
+    API_URL = "https://api.datacite.org"
+    
+    def wrap_event(event)
+      puts "------------------ \n"
+      totale = event.dig("totale").fetch("buckets", [])
+      # puts event.dig("unique").fetch("buckets", nil)
+      unique = event.dig("unique").fetch("buckets", [])
+      # puts unique[1].dig('key')
+    
+      unique_regular = unique.find_all {|access_method| access_method.fetch('key',"").match('regular') }
+      unique_machine = unique.find_all {|access_method| access_method.fetch('key',"").match('machine') }
+      total_regular  = totale.find_all {|access_method| access_method.fetch('key',"").match('regular') }
+      total_machine  = totale.find_all {|access_method| access_method.fetch('key',"").match('machine') }
+
+      dataset = { 
+        doi: event.dig("key","doi"), 
+        unique_counts_regular: unique_regular.empty? ? 0 : unique_regular.size,
+        unique_counts_machine: unique_machine.empty? ? 0 : unique_machine.size,
+        total_counts_regular: total_regular.empty? ? 0 : total_regular.dig(0,"doc_count"),
+        total_counts_machine: total_machine.empty? ? 0:  total_machine.dig(0,"doc_count")
+      }
+
+    
+      # conn = Faraday.new(:url => API_URL)
+      logger = Logger.new(STDOUT)
+      logger.info event.fetch("doc_count")
+      
+      # arr = dois.map do |dataset| 
+        logger.info dataset
+        doi = dataset.fetch(:doi,nil)
+        # json = conn.get "/works/#{doi}"
+        # json = conn.get do |req|
+        #   req.url "/works/#{doi}"
+        #   req.options.timeout = 50           # open/read timeout in seconds
+        #   req.options.open_timeout = 20      # connection open timeout in seconds
+        # end
+        # json = Maremma.get "#{API_URL}/works/#{doi}"
+        # logger.info json.status
+
+        # return {} unless json.status == 200 
+        # logger.info "Success on getting metadata for #{doi}"
+        # data = JSON.parse(json.body)
+        # data = json.body
+        data = {}
+        instances =[
+          {
+            count: dataset.fetch(:total_counts_regular),
+            "access-method": "regular",
+            "metric-type": "total-resolutions"
+          },
+          {
+            count: dataset.fetch(:unique_counts_regular),
+            "access-method": "regular",
+            "metric-type": "unique-resolutions"
+          },
+          {
+            count: dataset.fetch(:unique_counts_machine),
+            "access-method": "machine",
+            "metric-type": "unique-resolutions"
+          },
+          {
+            count: dataset.fetch(:total_counts_machine),
+            "access-method": "machine",
+            "metric-type": "total-resolutions"
+          },
+        ]
+    
+        instances.delete_if {|instance| instance.dig(:count) <= 0}
+        attributes = {} #data.dig("data","attributes")
+        resource_type = "" #attributes.fetch("resource-type-id",nil).nil? ? "dataset" : attributes.fetch("resource-type-id",nil)
+
+        instanced = { 
+          "dataset-id" => [{type: "doi", value: dataset.fetch(:doi,nil)}],
+          "data-type" => resource_type,
+          yop: attributes.fetch("published",nil),
+          uri: attributes.fetch("identifier",nil),
+          publisher: attributes.fetch("container-title",nil),
+          "dataset-title": attributes.fetch("title",nil),
+          "publisher-id": [{
+            type: "client-id",
+            value: attributes.fetch("data-center-id",nil)
+          }],
+          "dataset-dates": [{
+            type: "pub-date",
+            value: attributes.fetch("published",nil)
+          }],
+          "dataset-contributors": attributes.fetch("author",[]).map { |a| get_authors(a) },
+          platform: "datacite",
+          performance: [{
+            period: @period,
+            instance: instances
+          }]
+        }
+        logger.info instanced
+
+      instanced
+    end
+    
+    
+    def get_authors author
+      if (author.key?("given") && author.key?("family"))
+        { type: "name",
+          value: author.fetch("given",nil)+" "+author.fetch("family",nil) }
+        elsif author.key?("literal")
+          { type: "name",
+            value: author.fetch("literal",nil) }
+        else 
+          { type: "name",
+            value: "" }
+      end
+    end
+  
+  end
+end

data/lib/kishu/utils.rb

@@ -0,0 +1,115 @@
+require "bolognese"
+require "time"
+
+module Kishu
+  module Utils
+    include ::Bolognese::MetadataUtils
+    
+    def clean_tmp
+      system("rm tmp/datasets-*.json")
+      puts "/tmp Files deleted"
+    end
+
+    def merged_file
+      "reports/datacite_resolution_report_#{report_period.strftime("%Y-%m")}_2.json"
+    end
+
+    def encoded_file
+      "reports/datacite_resolution_report_#{report_period.strftime("%Y-%m")}_encoded.json"
+    end 
+
+    def generate_header_footer
+      report_header = '{"report-header": '+get_header.to_json.to_s+',"report-datasets": [ '+"\n"
+      
+      File.open("tmp/datasets-00-report-header.json","w") do |f|
+        f.write(report_header)
+      end
+      report_footer = ']'+"\n"+'}'
+      
+      File.open("tmp/datasets-zz99-report-footer.json","w") do |f|
+        f.write(report_footer)
+      end
+    end
+
+    def get_authors author
+      if (author.key?("given") && author.key?("family"))
+        { type: "name",
+          value: author.fetch("given",nil)+" "+author.fetch("family",nil) }
+        elsif author.key?("literal")
+          { type: "name",
+            value: author.fetch("literal",nil) }
+        else 
+          { type: "name",
+            value: "" }
+      end
+    end
+
+    def format_instance  data, options={}
+      obj = get_metadata(options[:dataset_id])
+      subj = {id:options[:report_id]}
+      # subj = "https://api.datacite.org/reports/0cb326d1-e3e7-4cc1-9d86-7c5f3d5ca310"
+      relation_type = "#{data[:"metric-type"]}-#{data[:"access-method"]}"
+      source_id = "datacite-resolution"
+      source_token = SOURCE_TOKEN
+      { 
+        "data" => {
+          "type" => "events",
+          "attributes" => {
+            "message-action" => "create",
+            "subj-id" => options[:report_id],
+            "total" => data[:count],
+            "obj-id" => options[:dataset_id],
+            "relation-type-id" => relation_type.to_s.dasherize,
+            "source-id" => source_id,
+            "source-token" => source_token,
+            "occurred-at" => Time.now.iso8601, # need modify
+            "timestamp" => Time.now.iso8601,
+            "license" => LICENSE,
+            "subj" => subj,
+            "obj" => obj } }}
+    end
+
+    def get_metadata id
+      doi = doi_from_url(id)
+      return {} unless doi.present?
+
+      url = API_URL + "/dois/#{doi}"
+      response = Maremma.get(url)
+      return {} if response.status != 200
+      
+      attributes = response.body.dig("data", "attributes")
+      relationships = response.body.dig("data", "relationships")
+  
+      resource_type = response.body.dig("data", "relationships")
+      resource_type_general = relationships.dig("resource-type", "data", "id")
+      type = Bolognese::Utils::CR_TO_SO_TRANSLATIONS[resource_type.to_s.underscore.camelcase] || Bolognese::Utils::DC_TO_SO_TRANSLATIONS[resource_type_general.to_s.underscore.camelcase(first_letter = :upper)] || "CreativeWork"
+      author = Array.wrap(attributes["author"]).map do |a| 
+        {
+          "given_name" => a["givenName"],
+          "family_name" => a["familyName"],
+          "name" => a["familyName"].present? ? nil : a["name"] }.compact
+      end
+      client_id = relationships.dig("client", "data", "id")
+  
+      {
+        "id" => id,
+        "type" => type.underscore.dasherize,
+        "name" => attributes["title"],
+        "author" => author,
+        "publisher" => attributes["publisher"],
+        "version" => attributes["version"],
+        "date_published" => attributes["published"],
+        "date_modified" => attributes["updated"],
+        "registrant_id" => "datacite.#{client_id}" }.compact
+    end
+
+    def encoded
+      Base64.strict_encode64(compress_merged_file)
+    end
+
+    def checksum
+      Digest::SHA256.hexdigest(compress_merged_file)
+    end
+
+  end
+end

data/lib/kishu/version.rb

@@ -0,0 +1,3 @@
+module Kishu
+  VERSION = "0.0.0"
+end

data/spec/factories/default.rb

@@ -0,0 +1,71 @@
+
+FactoryBot.define do
+  factory :resolution_event do
+    period  "begin_date": "2018-03-01", "end_date": "2018-03-31"
+    event  
+    {
+      "key": "10.5065/D6V1236Q",
+      "doc_count": 5566,
+      "total": {
+        "doc_count_error_upper_bound": 0,
+        "sum_other_doc_count": 0,
+        "buckets": [
+          {
+            "key": "machine",
+            "doc_count": 5093
+          },
+          {
+            "key": "regular",
+            "doc_count": 473
+          }
+        ]
+      },
+      "access_method": {
+        "doc_count_error_upper_bound": 0,
+        "sum_other_doc_count": 0,
+        "buckets": [
+          {
+            "key": "machine",
+            "doc_count": 5093,
+            "session": {
+              "doc_count_error_upper_bound": 10,
+              "sum_other_doc_count": 5072,
+              "buckets": [
+                {
+                  "key": "2018-09-18_16_10.5065/D6V1236Q_54.71.12.185_curl/7.38.0",
+                  "doc_count": 5
+                },
+                {
+                  "key": "2018-09-01_05_10.5065/D6V1236Q_45.79.139.170_curl/7.38.0",
+                  "doc_count": 4
+                },
+                {
+                  "key": "2018-09-03_16_10.5065/D6V1236Q_52.40.104.81_curl/7.38.0",
+                  "doc_count": 4
+                },
+                {
+                  "key": "2018-09-12_00_10.5065/D6V1236Q_52.39.7.168_curl/7.38.0",
+                  "doc_count": 4
+                },
+                {
+                  "key": "2018-09-26_06_10.5065/D6V1236Q_52.39.7.168_curl/7.38.0",
+                  "doc_count": 4
+                }
+              ]
+            },
+            "unqiue": {
+              "value": 3084
+            }
+          }]  
+      }
+    }
+  end
+
+
+  factory :usage_event do
+  end
+
+  factory :report do
+  end
+
+end