libis-rosetta_checker 1.0.2 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 40863874b758fc68bd975d1fec901f82f9631b76
-  data.tar.gz: 6a2276c3c7ad35743c0c849c6b216557d32d5168
+  metadata.gz: 8586a77252fe8f251c32bb348149b91a68af8ccc
+  data.tar.gz: f89774bd8dadd489dbf67df4ca5f025b05da9bf8
 SHA512:
-  metadata.gz: 5fd604dcd4264eccb3ccf862fe491e75f3d581994e13f26b02a98a676b88cbeed657543acb2c8e0f480f46f59297f8e0ffda8ac22fc0fa699c09f9d04aba9b91
-  data.tar.gz: 6078eda5e7ee0cefeb7e498af8f271a76d95d1bc3f7061d69add24dfb085eca0f815cce7f0bbea74d1e8650e045456d7408c0f1b7f5d2d2853f91fc042ec2db3
+  metadata.gz: 3efb31f23bb3ef27d436f8742c3466947ee8616269ff9466cc5bfacc3f21e560e1a8d9c2cd0999a4357a042e470c2f0d662eb98f3fc8c5a8c239507a3dafcacb
+  data.tar.gz: a9ebe6db7c594e19a0e2ea6bfa587ba8cbe20bc850dd7e2a3492bdb518c887a082519ff1897fd9d2ecc8872872638d90f7e9c396342a2882cc9a88297f66834e

data/lib/libis/rosetta_checker/files_to_ingest_cleanup.rb

@@ -59,8 +59,8 @@ module Libis
 
       protected
 
-      SQL_DATA = %w'ie_id rep_id fl_id original_name owner label group_id entity_type user_c'
-      CSV_HEADER = %w'parent_type parent file size md5 found name_match' + SQL_DATA
+      SQL_DATA = %w'ie_id rep_id fl_id original_name owner label group_id entity_type user_c path'
+      CSV_HEADER = %w'parent_type parent file size md5 sha1 found name_match' + SQL_DATA
 
       LOG_PATTERN = "[%d #%p] %-5l : %m\n".freeze
 
@@ -82,7 +82,9 @@ module Libis
           ci.LABEL as label,
           cf.GROUPID as group_id,
         	ci.ENTITYTYPE as entity_type,
-        	ci.PARTITIONC as user_c
+        	ci.PARTITIONC as user_c,
+        	pm.MID AS mid,
+        	md.VALUE AS dnx
         FROM
         	V2KU_PER00.PERMANENT_INDEX ps
         	LEFT JOIN V2KU_SHR00.STORAGE_PARAMETER sp ON sp.STORAGE_ID = ps.STORAGE_ID
@@ -90,6 +92,8 @@ module Libis
         	LEFT JOIN V2KU_REP00.HDECONTROL cf ON cf.PID = ps.STORED_ENTITY_ID
         	LEFT JOIN V2KU_REP00.HDECONTROL cr ON cr.PID = cf.PARENTID
         	LEFT JOIN V2KU_REP00.HDECONTROL ci ON ci.PID = cr.PARENTID
+        	LEFT JOIN V2KU_REP00.HDEPIDMID pm ON pm.PID = cf.PID
+        	LEFT JOIN V2KU_REP00.HDEMETADATA md ON md.MID = pm.MID
         WHERE
         	sp."KEY" = 'DIR_ROOT'
         AND	ps.FILE_SIZE = :filesize
@@ -98,6 +102,7 @@ module Libis
         AND cf.OBJECTTYPE = 'FILE'
         AND cr.OBJECTTYPE = 'REPRESENTATION'
         AND ci.OBJECTTYPE = 'INTELLECTUAL_ENTITY'
+        AND md.mdid = 21
       SQL
 
       def setup_logging
@@ -189,19 +194,11 @@ module Libis
         logger.info "- #{file}"
         if File.extname(file) == '.bz2'
           logger.info MSG_DEFLATE
-          info[:size] = 0
-          reader = Bzip2::FFI::Reader.open file
-          md5 = Digest::MD5.new
-          logger.info MSG_CALC_FC
-          while (data = reader.read 2048000) do
-            info[:size] += data.length
-            md5 << data
-          end
-          reader.close
           info[:parent_type] = 'F'
           info[:parent] = file
           info[:file] = File.basename file, '.bz2'
-          info[:md5] = md5.hexdigest
+          logger.info MSG_CALC_FC
+          checksum_file Bzip2::FFI::Reader.open(file), info
           check_file info
         elsif File.extname(file) == '.zip'
           logger.info '  - Unpacking'.freeze
@@ -213,23 +210,14 @@ module Libis
               info[:file] = entry.name
               logger.info "- #{file}/#{entry.name}"
               logger.info MSG_CALC_FC
-              info[:size] = 0
-              md5 = Digest::MD5.new
-              reader = entry.get_input_stream
-              while (data = reader.read 2048000) do
-                info[:size] += data.length
-                md5 << data
-              end
-              reader.close
-              info[:md5] = md5.hexdigest
+              checksum_file entry.get_input_stream, info
               check_file info
             end
           end
         else
           begin
             logger.info MSG_CALC_FC
-            info[:size] = File.size file
-            info[:md5] = Digest::MD5.file file
+            checksum_file File.open(file), info
             check_file info
           rescue Exception
             logger.error "Could not access file '#{file}'"
@@ -237,6 +225,20 @@ module Libis
         end
       end
 
+      def checksum_file(reader, info)
+        info[:size] = 0
+        md5 = Digest::MD5.new
+        sha1 = Digest::SHA1.new
+        while (data = reader.read 2048000) do
+          info[:size] += data.length
+          md5 << data
+          sha1 << data
+        end
+        reader.close
+        info[:md5] = md5.hexdigest
+        info[:sha1] = sha1.hexdigest
+      end
+
       def check_file(info)
         logger.info MSG_CHCK_DB
 
@@ -245,9 +247,12 @@ module Libis
 
         cursor.exec
 
+        info[:found] = 0
         info_list = []
 
         while (found = cursor.fetch_hash)
+          dnx = found['DNX'].read
+          next unless dnx =~ /<key id="fixityType">SHA1<\/key><key id="fixityValue">#{info[:sha1]}<\/key>/
           SQL_DATA.each {|x| info[x.to_sym] = found[x.upcase]}
           logger.info "    found match: #{info[:ie_id]}/#{info[:rep_id]}/#{info[:fl_id]}"
           if info[:original_name] =~ Regexp.new(info[:file].split(/[ #._-]/).join('.*'))
@@ -261,15 +266,13 @@ module Libis
 
         end
 
-        case cursor.row_count
-          when 0..1
-            info[:found] = cursor.row_count
-            to_report info
-          else
-            info_list.each do |i|
-              i[:found] = cursor.row_count
-              to_report i
-            end
+        if info_list.empty?
+          to_report info
+        else
+          info_list.each do |i|
+            i[:found] = info_list.count
+            to_report i
+          end
         end
 
       end

data/lib/libis/rosetta_checker/version.rb

@@ -1,5 +1,5 @@
 module Libis
   module RosettaChecker
-    VERSION = "1.0.2"
+    VERSION = "1.0.3"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: libis-rosetta_checker
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.3
 platform: ruby
 authors:
 - Kris Dekeyser
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-12-20 00:00:00.000000000 Z
+date: 2018-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby-oci8