ndr_pseudonymise 0.4.1

data/lib/ndr_pseudonymise/ndr_encrypt/encrypted_object.rb

@@ -0,0 +1,124 @@
+require 'digest'
+require 'io/console'
+require 'openssl'
+require 'zlib'
+
+module NdrPseudonymise
+  module NdrEncrypt
+    # Defines utility methods for encrypting / decrypting objects
+    module EncryptedObject
+      # rubocop:disable Style/SlicingWithRange
+      def self.blob(data)
+        "blob #{data.size}\0#{data}"
+      end
+
+      def self.unpack_blob(blob)
+        prefix, data = blob.split("\x00", 2)
+        raise(ArgumentError, 'Invalid blob format') unless /\Ablob [0-9]+\z/ =~ prefix
+
+        size = prefix[5..-1].to_i
+        raise(ArgumentError, 'Incorrect blob size') unless size == data.size
+
+        data
+      end
+
+      def self.digest(blob)
+        Digest::SHA256.hexdigest(blob)
+      end
+
+      # Create zlib-compressed version of the content
+      def self.compress(blob)
+        Zlib::Deflate.deflate(blob)
+      end
+
+      # Unpack zlib-compressed content
+      def self.decompress(contents)
+        Zlib::Inflate.inflate(contents)
+      end
+
+      def self.encrypted_id(git_blobid, key_name: nil)
+        # We need to support ruby 2.0 so cannot use required keyword arguments syntax
+        raise(ArgumentError, 'missing keyword: :key_name') unless key_name
+
+        temp_id = "ndr_encrypt #{git_blobid} #{key_name}"
+        digest(blob(temp_id))
+      end
+
+      # Encrypt sensitive secret data, given a public key file as a String
+      # Returns the encrypted output data
+      # Result can either be decrypted using the decrypt method on this class.
+      # TODO: write equivalent command-line method using only openssl and shell scripts
+      def self.encrypt(secret_data, pub_key: nil)
+        # We need to support ruby 2.0 so cannot use required keyword arguments syntax
+        raise(ArgumentError, 'missing keyword: :pub_key') unless pub_key
+        return nil unless secret_data
+
+        public_key_data = File.read(pub_key)
+        cipher = OpenSSL::Cipher.new('aes-256-cbc')
+        cipher.encrypt
+        cipher.key = random_key = cipher.random_key
+        cipher.iv = random_iv = cipher.random_iv
+        rawdata = cipher.update(secret_data)
+        rawdata << cipher.final
+        public_key = OpenSSL::PKey::RSA.new(public_key_data)
+        public_key.public_encrypt(random_key) + random_iv + rawdata
+      end
+
+      # Decrypt sensitive secret data, given a private key and its password
+      # Returns the decrypted output data
+      # TODO: write equivalent command-line method using only openssl and shell scripts
+      # TODO: Refactor with code from era UnifiedSources::ApiRetrieval::Extractor
+      def self.decrypt(rawdata, private_key: nil, passin: nil)
+        # We need to support ruby 2.0 so cannot use required keyword arguments syntax
+        raise(ArgumentError, 'missing keyword: :private_key') unless private_key
+        return nil unless rawdata
+
+        password = get_passphrase(private_key: private_key, passin: passin)
+        private_key_data = File.read(private_key)
+        cipher = OpenSSL::Cipher.new('aes-256-cbc')
+        cipher.decrypt
+        private_key = OpenSSL::PKey::RSA.new(private_key_data, password)
+        key_size = private_key.n.num_bytes
+        cipher.key = private_key.private_decrypt(rawdata[0..key_size - 1])
+        cipher.iv = rawdata[key_size..key_size + 15]
+        decrypted_data = cipher.update(rawdata[key_size + 16..-1])
+        decrypted_data << cipher.final
+      end
+
+      def self.get_passphrase(private_key: nil, passin: nil)
+        # We need to support ruby 2.0 so cannot use required keyword arguments syntax
+        raise(ArgumentError, 'missing keyword: :private_key') unless private_key
+
+        @passphrase_cache ||= {}
+        return @passphrase_cache[private_key] if @passphrase_cache.key?(private_key)
+
+        raise(ArgumentError, 'Missing private key file') unless File.exist?(private_key)
+
+        # Implement a subset of the openssl -passin options in
+        # https://www.openssl.org/docs/man3.0/man1/openssl-passphrase-options.html
+        result = case passin
+                 when nil, ''
+                   msg = "Enter passphrase for #{private_key}: "
+                   if IO.console.respond_to?(:getpass)
+                     IO.console.getpass msg
+                   else
+                     $stdout.print msg
+                     password = $stdin.noecho(&:gets).chomp
+                     puts
+                     password
+                   end
+                 when /\Apass:/
+                   passin[5..-1]
+                 when /\Aenv:/
+                   ENV[passin[4..-1]]
+                 when 'stdin'
+                   $stdin.readline.chomp
+                 else
+                   raise(ArgumentError, 'Unsupported passin option')
+                 end
+        @passphrase_cache[private_key] = result
+      end
+      # rubocop:enable Style/SlicingWithRange
+    end
+  end
+end

data/lib/ndr_pseudonymise/ndr_encrypt/remote_repository.rb

@@ -0,0 +1,44 @@
+require 'net/http'
+require 'uri'
+
+module NdrPseudonymise
+  module NdrEncrypt
+    # Defines a local ndr_encrypt working copy
+    class RemoteRepository
+      # rubocop:disable Style/SlicingWithRange
+      def initialize(base_url: nil)
+        # We need to support ruby 2.0 so cannot use required keyword arguments syntax
+        raise(ArgumentError, 'missing keyword: :base_url') unless base_url
+
+        @base_url = base_url
+      end
+
+      # Retrieve remote file(s) based on git_blobid
+      def cat_remote(git_blobid, key_name: nil, private_key: nil, passin: nil)
+        # We need to support ruby 2.0 so cannot use required keyword arguments syntax
+        raise(ArgumentError, 'missing keyword: :key_name') unless key_name
+        raise(ArgumentError, 'missing keyword: :private_key') unless private_key
+
+        encrypted_id = NdrEncrypt::EncryptedObject.encrypted_id(git_blobid, key_name: key_name)
+        rawdata = retrieve_remote_url(encrypted_id)
+        contents = NdrEncrypt::EncryptedObject.decrypt(rawdata, private_key: private_key,
+                                                                passin: passin)
+        blob = NdrEncrypt::EncryptedObject.decompress(contents)
+        NdrEncrypt::EncryptedObject.unpack_blob(blob)
+      end
+
+      private
+
+      # Retrieve remote encrypted file(s) based on encrypted_id
+      def retrieve_remote_url(encrypted_id)
+        uri = URI.join(@base_url, "#{encrypted_id[0..1]}/#{encrypted_id[2..-1]}")
+        res = Net::HTTP.get_response(uri)
+        # TODO: More finegrained error messages
+        raise(ArgumentError, 'Could not retrieve URL') unless res.is_a?(Net::HTTPSuccess)
+
+        res.body
+      end
+      # rubocop:enable Style/SlicingWithRange
+    end
+  end
+end

data/lib/ndr_pseudonymise/ndr_encrypt/repository.rb

@@ -0,0 +1,165 @@
+require 'csv'
+require 'fileutils'
+require 'set'
+require 'stringio'
+
+module NdrPseudonymise
+  module NdrEncrypt
+    # Defines a local ndr_encrypt working copy
+    class Repository
+      # rubocop:disable Style/SlicingWithRange
+      CSV_COLUMNS = %w[git_blobid path].freeze
+      ENCRYPTED_DIR = 'ndr_encrypted/'.freeze
+
+      def initialize(repo_dir: nil)
+        # We need to support ruby 2.0 so cannot use required keyword arguments syntax
+        raise(ArgumentError, 'missing keyword: :repo_dir') unless repo_dir
+
+        @repo_dir = repo_dir
+      end
+
+      # Create directory structure
+      def init
+        FileUtils.mkdir_p(object_dir)
+        return false if valid_repository?
+
+        CSV.open(index_filename, 'wb') { |csv| csv << CSV_COLUMNS }
+        true
+      end
+
+      # Add file contents to the encrypted store and index
+      def add(paths, key_name: nil, pub_key: nil)
+        # We need to support ruby 2.0 so cannot use required keyword arguments syntax
+        raise(ArgumentError, 'missing keyword: :key_name') unless key_name
+        raise(ArgumentError, 'missing keyword: :pub_key') unless pub_key
+        raise(ArgumentError, 'Invalid ndr_encrypted encrypted store') unless valid_repository?
+
+        paths.each do |path|
+          git_blobid, _encrypted_id = hash_object(path,
+                                                  key_name: key_name, pub_key: pub_key, write: true)
+          File.open(index_filename, 'ab') { |f| f << [git_blobid, path].to_csv }
+        end
+      end
+
+      # Cleanup unnecessary index entries and optimize the encrypted store
+      def gc(output_stream: StringIO.new)
+        raise(ArgumentError, 'Invalid ndr_encrypted encrypted store') unless valid_repository?
+
+        output_stream.print('Reading index: ')
+        csv_data = CSV.read(index_filename)
+        header = csv_data.shift
+        raise(ArgumentError, 'Invalid header in index file') unless CSV_COLUMNS == header
+
+        count0 = csv_data.size
+        output_stream.print("#{count0} entries.\nRemoving duplicates: ")
+        csv_data.each.with_index do |row, i|
+          unless row.size == 2 && row[0] =~ /\A[0-9a-f]+\z/
+            raise(ArgumentError, "Invalid index entry on data row #{i + 1}")
+          end
+        end
+        csv_data = csv_data.sort.uniq
+        count1 = csv_data.size
+        output_stream.print("#{count1} entries remaining.\nWriting objects: ")
+        # Move aside index file temporarily to reduce race conditions
+        # Note: should use a proper lock file for all index interactions
+        orig_filename = "#{index_filename}.orig"
+        temp_filename = "#{index_filename}.new"
+        FileUtils.mv(index_filename, "#{index_filename}.orig")
+        CSV.open(temp_filename, 'wb') do |csv|
+          csv << header
+          csv_data.each { |row| csv << row }
+        end
+        FileUtils.mv(temp_filename, index_filename)
+        FileUtils.rm(orig_filename)
+        output_stream.puts("100% (#{count1}/#{count1}), done.\n")
+        output_stream.puts("Total #{count1} (delta #{count0 - count1})")
+      end
+
+      # Retrieve local file(s) based on CSV entry
+      def get(paths, key_name: nil, private_key: nil, passin: nil)
+        # We need to support ruby 2.0 so cannot use required keyword arguments syntax
+        raise(ArgumentError, 'missing keyword: :key_name') unless key_name
+        raise(ArgumentError, 'missing keyword: :private_key') unless private_key
+        raise(ArgumentError, 'Invalid ndr_encrypted encrypted store') unless valid_repository?
+
+        path_set = Set.new(paths)
+        paths = path_set.to_a # Keep only unique entries
+        found = Set.new # index may have duplicate objects if not garbage collected
+        CSV.foreach(index_filename, headers: true) do |row|
+          # Only keep first matching entry for each path
+          if path_set.include?(row['path'])
+            found << row
+            path_set.delete(row['path'])
+            break if path_set.empty?
+          end
+        end
+        raise(ArgumentError, 'Cannot find some files') unless found.size == paths.size
+
+        found.each do |row|
+          data = cat_file(row['git_blobid'], key_name: key_name, private_key: private_key,
+                                             passin: passin)
+          File.open(row['path'], 'wb') { |f| f << data }
+        end
+      end
+
+      # Compute object IDs and optionally creates an encrypted object from a file
+      # Returns [git_blobid, encrypted_id]
+      def hash_object(path, key_name: nil, pub_key: nil, write: nil)
+        # We need to support ruby 2.0 so cannot use required keyword arguments syntax
+        raise(ArgumentError, 'missing keyword: :key_name') unless key_name
+        raise(ArgumentError, 'missing keyword: :pub_key') unless pub_key
+
+        data = File.binread(path)
+        blob = NdrEncrypt::EncryptedObject.blob(data)
+        git_blobid = NdrEncrypt::EncryptedObject.digest(blob)
+        encrypted_id = NdrEncrypt::EncryptedObject.encrypted_id(git_blobid, key_name: key_name)
+        if write
+          encrypted_dir = File.join(object_dir, encrypted_id[0..1])
+          encrypted_filename = File.join(encrypted_dir, encrypted_id[2..-1])
+          unless File.exist?(encrypted_filename) # Don't override existing file
+            contents = NdrEncrypt::EncryptedObject.compress(blob)
+            encrypted_contents = NdrEncrypt::EncryptedObject.encrypt(contents, pub_key: pub_key)
+            FileUtils.mkdir_p(encrypted_dir)
+            File.open(encrypted_filename, 'wb') { |f| f << encrypted_contents }
+          end
+        end
+        [git_blobid, encrypted_id]
+      end
+
+      # Retrieve local file(s) based on git_blobid
+      def cat_file(git_blobid, key_name: nil, private_key: nil, passin: nil)
+        # We need to support ruby 2.0 so cannot use required keyword arguments syntax
+        raise(ArgumentError, 'missing keyword: :key_name') unless key_name
+        raise(ArgumentError, 'missing keyword: :private_key') unless private_key
+
+        encrypted_id = NdrEncrypt::EncryptedObject.encrypted_id(git_blobid, key_name: key_name)
+        encrypted_filename = File.join(object_dir, encrypted_id[0..1], encrypted_id[2..-1])
+        unless File.exist?(encrypted_filename)
+          raise(ArgumentError, 'File does not exist in encrypted storage')
+        end
+
+        rawdata = File.binread(encrypted_filename)
+        contents = NdrEncrypt::EncryptedObject.decrypt(rawdata, private_key: private_key,
+                                                                passin: passin)
+        blob = NdrEncrypt::EncryptedObject.decompress(contents)
+        NdrEncrypt::EncryptedObject.unpack_blob(blob)
+      end
+
+      private
+
+      # Does the repository have a valid structure
+      def valid_repository?
+        Dir.exist?(object_dir) && File.exist?(index_filename)
+      end
+
+      def object_dir
+        File.join(@repo_dir, ENCRYPTED_DIR, 'objects')
+      end
+
+      def index_filename
+        File.join(@repo_dir, ENCRYPTED_DIR, 'index.csv')
+      end
+      # rubocop:enable Style/SlicingWithRange
+    end
+  end
+end

data/lib/ndr_pseudonymise/ndr_encrypt.rb

@@ -0,0 +1,10 @@
+require 'ndr_pseudonymise/ndr_encrypt/command_line'
+require 'ndr_pseudonymise/ndr_encrypt/encrypted_object'
+require 'ndr_pseudonymise/ndr_encrypt/remote_repository'
+require 'ndr_pseudonymise/ndr_encrypt/repository'
+
+module NdrPseudonymise
+  # Utilities and a command line tool for an encrypted object store
+  module NdrEncrypt
+  end
+end

data/lib/ndr_pseudonymise/prescription_pseudonymiser.rb

@@ -0,0 +1,71 @@
+# Fast, simple pseudonymisation of prescription data with a very controlled
+# format.
+# Only the first 2 fields are potentially identifiable: nhs number and date of
+# birth.
+
+require 'ndr_pseudonymise/simple_pseudonymisation'
+require 'ndr_pseudonymise/pseudonymisation_specification'
+
+require 'json'
+
+module NdrPseudonymise
+  # Pseudonymise prescription data
+  class PrescriptionPseudonymiser < PseudonymisationSpecification
+    PREAMBLE_V2_DEMOG_ONLY = 'Pseudonymised matching data v2.0-demog-only'.freeze
+
+    def initialize(format_spec, key_bundle)
+      super
+      return if @format_spec[:demographics] == [0, 1]
+      raise 'Invalid specification: expected nhsnumber and birthdate in first 2 columns'
+    end
+
+    # Validate a row of prescription data
+    # Return false if this row is a valid data row, otherwise a list of errors
+    def row_errors2(row)
+      # Not significantly faster than optimised general #row_errors method
+      (nhsnumber, birthdate) = row[0..1]
+      unless nhsnumber.is_a?(String) && nhsnumber =~ /\A([0-9]{10})?\Z/
+        raise 'Invalid NHS number'
+      end
+      raise 'Missing NHS number' if nhsnumber.size < 10
+      unless birthdate.is_a?(String) && birthdate =~ /\A([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]|)\Z/
+        raise 'Invalid birthdate'
+      end
+    end
+
+    # Pseudonymise a row of prescription data, returning an array of a single row:
+    # [[packed_pseudoid_and_demographics, clinical_data1, ...]]
+    # Where packed_pseudoid_and_demographics consists of
+    # "pseudo_id1 (key_bundle) packed_pseudoid_and_demographics"
+    def pseudonymise_row(row)
+      @key_cache ||= {} # Cache pseudonymisation keys for more compact import
+      all_demographics = { 'nhsnumber' => row[0], 'birthdate' => row[1] }
+      key = all_demographics.to_json
+      if @key_cache.key?(key)
+        pseudo_id1, key_bundle, demog_key = @key_cache[key]
+      else
+        pseudo_id1, key_bundle, demog_key = NdrPseudonymise::SimplePseudonymisation.
+                                            generate_keys_nhsnumber_demog_only(@salt1, @salt2, row[0])
+        if !row[0].to_s.empty? && !row[1].to_s.empty? # && false to stop caching
+          @key_cache = {} if @key_cache.size > 10000 # Limit cache size
+          @key_cache[key] = [pseudo_id1, key_bundle, demog_key]
+        end
+      end
+      encrypted_demographics = NdrPseudonymise::SimplePseudonymisation.
+                               encrypt_data64(demog_key, all_demographics.to_json)
+      packed_pseudoid_and_demographics = format('%s (%s) %s', pseudo_id1, key_bundle,
+                                                encrypted_demographics)
+      [[packed_pseudoid_and_demographics] + row[2..-1]]
+    end
+
+    # Header row for CSV data
+    def csv_header_row
+      [PREAMBLE_V2_DEMOG_ONLY]
+    end
+
+    # Append the output of pseudonymise_row to a CSV file
+    def emit_csv_rows(out_csv, pseudonymised_row)
+      out_csv << pseudonymised_row[0]
+    end
+  end
+end

data/lib/ndr_pseudonymise/progress_printer.rb

@@ -0,0 +1,53 @@
+module NdrPseudonymise
+  # Log percentage progress on pseudonymisation
+  # Starts logging after 1 minute or 5%, then at 5% / 5 minute intervals
+  class ProgressPrinter
+    # Logs progress to the given stream (default $stdout)
+    # If verbose = false, only log percentages on a single line
+    # If verbose = true, log verbose output
+    # If verbose = :dynamic, act like verbose = false, but if the total time is
+    # more than 5 minutes, move into verbose = true mode
+    def initialize(dest = $stdout, verbose = false)
+      @dest = dest
+      @verbose = verbose
+      @last_percent = 0
+      @last_log = Time.current - (60 * 4) # First log entry after 1 minute
+    end
+
+    # Returns a lambda that prints progress to stdout (or another stream).
+    # parameter _csv_row is not used.
+    def log_progress(start_time, time_now, _csv_row, progress, total)
+      current_percentage = total == 0 ? 0 : (progress * 100 / total).to_i
+      now = Time.current
+      if (current_percentage / 5 > @last_percent / 5) || # Log at 5% / 5 minute intervals
+         (now - @last_log >= 60 * 5) || current_percentage == 100
+        if @verbose == :dynamic && (time_now - start_time >= 60 * 5)
+          @verbose = true
+          @dest << '...'
+        end
+        if @verbose == true
+          # TODO: Add estimated completion time
+          tfin = if progress > 0
+                   time_now + (time_now - start_time) * (total - progress) / progress
+                 end
+          completion = tfin ? ', expected completion' : ''
+          @dest << format("Completed %s%% in %.1f minutes%s\n",
+                          current_percentage, (now - start_time) / 60.0, completion)
+
+          # @dest << ("Completed %s%% in %.1f minutes#{", expected completion #{tfin}" if tfin}\n" %
+          #            [current_percentage, (now - start_time) / 60.0])
+
+        else
+          @dest << "#{'...' if @last_percent > 0}#{current_percentage}%"
+          @dest << "\n" if current_percentage == 100
+        end
+        # if current_percentage == 100 # Uncomment for performance debugging
+        #   @dest << "Finished %s rows in %.3f secs\n" % [csv_row, time_now - start_time]
+        # end
+        @dest.flush
+        @last_percent = current_percentage
+        @last_log = now
+      end
+    end
+  end
+end