ndr_pseudonymise 0.4.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: cdbd09c95ec2fa5a5fdb791a177654ae9c9cc520a1461c1df54cc5035c001c66
+  data.tar.gz: 3bb60274d7eed27d033bb7efca0262d48c0e8d322b6708eb0d6ddb37a9673a2a
+SHA512:
+  metadata.gz: 7180d2bf8a99f0b5493e0204bf9f3dff7100f8663f1560b2b207a05fa235431674b60490093cbd6eeef5e350ad7460f470ad895d71d3506b71d73dd29e52ad3b
+  data.tar.gz: 3261dcbbd12a6770d7d03e76bb84aa6e327de4b0acaec762e0650e5d8ba4425f5780ef051e8ef2a7063d5327ba8e28b1539ad4587520d0484d239377916d1555

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.rubocop-https--*
+/.yardoc
+/Gemfile.lock
+/gemfiles/*.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rubocop.yml

@@ -0,0 +1,2 @@
+require:
+  - ndr_dev_support

data/CHANGELOG.md

@@ -0,0 +1,44 @@
+## [Unreleased]
+* No unreleased changes
+
+## 0.4.1 / 2022-10-18
+## Added
+# Add ndr_encrypt utility script for image encryption.
+
+## 0.4.0 / 2020-07-08
+## Changed
+* Update client to reflect API updates
+
+## 0.3.0 / 2020-07-08
+## Added
+* Added pseudonymisation service client
+
+## 0.2.11 / 2019-03-15
+## Added
+* SimplePseudonymisation should include decryption methods.
+
+# 0.2.10 / 2019-03-06
+## Fixed
+* Pseudonymisation: allow blank dates of birth.
+
+# 0.2.9 / 2018-12-10
+## Changed
+* # Include row numbers in CSV pseudonymisation errors.
+
+# 0.2.8 / 2018-12-07
+## Fixed
+* Allow null NHS numbers for DemographicsOnlyPseudonymiser
+
+# 0.2.7 / Unreleased
+
+# 0.2.6 / 2018-08-30
+## Fixed
+* Remove unnecessary pry runtime dependency
+
+# 0.2.5 / 2018-07-24
+## Added
+* Support pseudonymising data with multiple demographics columns, and various date formats
+
+# 0.2.4 / 2018-07-24
+## Added
+* Add helper methods for reformatting pseudonymised data into ordinary CSV columns.

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in ndr_pseudonymise.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2011-2016 Public Health England
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,64 @@
+## NdrPseudonymise [![Build Status](https://github.com/publichealthengland/ndr_pseudonymise/workflows/Test/badge.svg)](https://github.com/publichealthengland/ndr_pseudonymise/actions?query=workflow%3Atest)
+
+Pseudonymise confidential data, in CSV format, with specifications for which fields to be encrypted.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'ndr_pseudonymise'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install ndr_pseudonymise
+
+## Usage
+
+Example input file:
+nhsnumber,birthdate,postcode,surname,data1,date2
+1234567881,1955-01-01,CB22 3AD,SMITH,xyz,abc
+,01/02/1955,,JONES,zzz,aaa
+
+outfile.yml:
+
+File version:
+'something-1'
+
+Pseudonymised keys:
+[sha1('nhsnumber_1234567881' + salt1), 'n-random-uuencoded-bits-1']
+[sha1('birthdate_postcode_1955-01-01_CB22 3AD' + salt1), 'n-random-uuencoded-bits-2']
+[sha1('birthdate_postcode_1955-02-01_' + salt1), 'n-random-uuencoded-bits-3']
+
+Encrypted demographics
+[sha1('nhsnumber_1234567881' + salt1), encrypt(['surname' => 'SMITH'], 'nhsnumber_1234567881' + 'n-random-uuencoded-bits-1' + salt2, 'n-random-uuencoded-bits2-1']
+[sha1('birthdate_postcode_1955-01-01_CB22 3AD'  + salt1), encrypt(['surname' => 'SMITH'], 'birthdate_postcode_1955-01-01_CB22 3AD'  + 'n-random-uuencoded-bits-2' + salt2, 'n-random-uuencoded-bits2-2']
+[sha1('birthdate_postcode_1955-02-01_' + salt1), encrypt(['surname' => 'JONES'], 'birthdate_postcode_1955-02-01_'  + 'n-random-uuencoded-bits-2' + salt2, 'n-random-uuencoded-bits2-3']
+
+Encrypted data
+[sha1('nhsnumber_1234567881' + salt1), encrypt(['xyz','abc'], 1', 'nhsnumber_1234567881' + 'n-random-uuencoded-bits2-1' + salt2)]
+[sha1('birthdate_postcode_1955-01-01_CB22 3AD' + salt1), encrypt(['xyz','abc'], 1', 'birthdate_postcode_1955-01-01_CB22 3AD' + 'n-random-uuencoded-bits2-2' + salt2)]
+[sha1('birthdate_postcode_1955-02-01_' + salt1), encrypt(['zzz','aaa'], 1', 'birthdate_postcode_1955-02-01_' + 'n-random-uuencoded-bits2-3' + salt2)]
+
+Encrypted meta-data, header row?
+
+TODO: Replace text e.g. 'nhsnumber_1234567881' on RHS above with e.g. sha1('extraprefix_' + 'nhsnumber_1234567881') i.e. something non-disclosive, derivable from the original data.
+TODO: Put original versions of e.g. nhsnumber, birthdate, postcode into "Encrypted demographics"
+TODO: Maybe salt2 could be retained, to allow some fuzzy demographic matching, without the possibility of brute forcing the main identifiers??? Or maybe you can do pseudonymised matching without any salt...
+
+Open questions:
+Standard date / postcode normalisation
+Do you escape underscores in field values
+Do we need salt2, or re-use salt1?
+Can we remove n-random-uuencoded-bits2 into the pseudonymised keys hash?
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).

data/Rakefile

@@ -0,0 +1,14 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+require 'ndr_dev_support/tasks'
+# require 'ndr_pseudonymise/tasks'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'test'
+  t.libs << 'lib'
+  t.test_files = FileList['test/**/*_test.rb']
+  t.verbose = false
+  t.warning = false
+end
+
+task default: :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'ndr_pseudonymise'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require 'pry'
+# Pry.start
+
+require 'irb'
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/ndr_pseudonymise/client.rb

@@ -0,0 +1,115 @@
+require 'json'
+require 'net/http'
+
+module NdrPseudonymise
+  # A class to wrap interactions with a remote pseudonymisation service.
+  #
+  # Sample usage, against a local pseudonymisation service:
+  #
+  #   client = NdrPseudonymise::Client.new(
+  #     host: 'localhost', port: 3000, use_ssl: false,
+  #     token: 'your_name:some_token', context: 'just testing'
+  #   )
+  #
+  #   client.pseudonymise(identifiers: { nhs_number: '0123456789' })
+  #
+  class Client
+    attr_accessor :context, :host
+
+    def initialize(host:, token:, port: 443, use_ssl: true, root_path: '/api/v1', context: nil)
+      @host = Net::HTTP.new(host, port).tap { |http| http.use_ssl = use_ssl }
+      @header = "Token token=#{token.inspect}"
+      @root_path = root_path
+      @context = context
+    end
+
+    # Returns a list of pseudonymisation keys that the current user is able to use.
+    #
+    # Sample usage:
+    #
+    #   client.keys #=> [{name"=>"key one", "supported_variants"=>[1]}, ...]
+    #
+    def keys
+      get('/keys')
+    end
+
+    # Returns a list of variants that the current user is able to use.
+    #
+    # Sample usage:
+    #
+    #   client.variants #=> [{"variant"=>"1", "required_identifiers" => ["nhs_number"]}, ...]
+    #
+    def variants
+      get('/variants')
+    end
+
+    # Returns pseudonymised identifiers for the supplied identifiers.
+    # By default, the pseudonymisation service requests all ID variants
+    # from all available keys; this can be filtered by using `variants`
+    # and `key_names` respectively.
+    #
+    # Sample usage:
+    #
+    #   client.pseudonymise(identifiers: { nhs_number: '0123456789' }) #=>
+    #     [
+    #       { "key_name"=>"key one", "variant"=>1, "pseudoid"=>"b549ef342...", "identifiers"=>... },
+    #       { "key_name"=>"key two", "variant"=>1, "pseudoid"=>"0ebd91c13...", "identifiers"=>... },
+    #     ]
+    #
+    #   client.pseudonymise(identifiers: { postcode: 'CB22 3AD', birth_date: '2000-01-01' }, key_names: ['key two']) #=>
+    #     [
+    #       { "key_name"=>"key two", "variant"=>2, "pseudoid"=>"043d5fc1a...", "identifiers"=>... },
+    #     ]
+    #
+    def pseudonymise(identifiers:, key_names: [], variants: [], context: @context)
+      raise ArgumentError, 'you must supply context!' if context.blank?
+
+      data = { identifiers: identifiers, context: context }
+      data[:key_names] = key_names if key_names.present?
+      data[:variants] = variants if variants.present?
+
+      post('/pseudonymise', data)
+    end
+
+    private
+
+    def get(endpoint)
+      handle_response { request(build_get_request(endpoint)) }
+    end
+
+    def post(endpoint, params)
+      handle_response { request(build_post_request(endpoint, params)) }
+    end
+
+    delegate :request, to: :@host
+
+    def handle_response
+      response = yield
+      data = response.body.present? ? JSON.parse(response.body) : {}
+
+      case response.code.to_i
+      when 200
+        data
+      else
+        raise <<~MESSAGE
+          An error occured trying to use the pseudonymisation service. Details:
+          #{data.fetch('errors', []).join(', ')}
+        MESSAGE
+      end
+    end
+
+    def build_get_request(endpoint)
+      Net::HTTP::Get.new(@root_path + endpoint).tap do |request|
+        request['Authorization'] = @header
+      end
+    end
+
+    def build_post_request(endpoint, params)
+      Net::HTTP::Post.new(@root_path + endpoint).tap do |request|
+        request['Authorization'] = @header
+        request['Content-Type'] = 'application/json'
+        request.body = JSON.dump(params)
+      end
+    end
+  end
+end

data/lib/ndr_pseudonymise/demographics_only_pseudonymiser.rb

@@ -0,0 +1,88 @@
+# Fast, simple pseudonymisation of prescription data with a very controlled
+# format.
+# Only the first 2 fields are potentially identifiable: nhs number and date of
+# birth.
+
+require 'ndr_pseudonymise/simple_pseudonymisation'
+require 'ndr_pseudonymise/pseudonymisation_specification'
+
+require 'json'
+
+module NdrPseudonymise
+  # Pseudonymise prescription data
+  class DemographicsOnlyPseudonymiser < PseudonymisationSpecification
+    PREAMBLE_V2_DEMOG_ONLY = 'Pseudonymised matching data v2.0-demog-only'.freeze
+
+   # Pseudonymise a row of prescription data, returning an array of a single row:
+    # [[packed_pseudoid_and_demographics, clinical_data1, ...]]
+    # Where packed_pseudoid_and_demographics consists of
+    # "pseudo_id1 (key_bundle) packed_pseudoid_and_demographics"
+    def pseudonymise_row(row)
+      @key_cache ||= {} # Cache pseudonymisation keys for more compact import
+      # all_demographics = { 'nhsnumber' => row[0], 'birthdate' => row[1] }
+      all_demographics_hash = {}
+      demographics_cols = @format_spec[:demographics]
+      row.each_with_index do |x, i|
+        row_spec = @format_spec[:columns][i]
+        all_demographics_hash[row_spec[:title]] = x if demographics_cols.include?(i)
+      end
+
+      # TODO: Refactor date handling into parent class's all_demographics_hash method
+      demographics_cols = @format_spec[:demographics]
+      row.each_with_index do |x, i|
+        row_spec = @format_spec[:columns][i]
+        if row_spec[:canonical_title]
+          if x.present? && row_spec[:strptime] && row_spec[:strftime]
+            # :strptime can contain a String (single format) or Array (a list of formats)
+            # :strftime contains a single string format
+            datetime = false
+            [row_spec[:strptime]].flatten(1).each do |format|
+              begin
+                datetime = DateTime.strptime(x, format)
+                break
+              rescue ArgumentError # Keep trying after invalid date formats
+              end
+            end
+            raise ArgumentError.new('Invalid date') if datetime == false # No formats matched
+            val = datetime.strftime(row_spec[:strftime])
+          else
+            val = x
+          end
+          all_demographics_hash[row_spec[:canonical_title]] = val
+        end
+
+      end
+
+      # Ensure NHS number is empty string (expected by SimplePseudonymisation), not nil
+      all_demographics_hash['nhsnumber'] = all_demographics_hash['nhsnumber'].to_s
+      nhsnumber = all_demographics_hash['nhsnumber']
+      birthdate = all_demographics_hash['birthdate']
+      key = all_demographics_hash.to_json
+      if @key_cache.key?(key)
+        pseudo_id1, key_bundle, demog_key = @key_cache[key]
+      else
+        pseudo_id1, key_bundle, demog_key = NdrPseudonymise::SimplePseudonymisation.
+                                            generate_keys_nhsnumber_demog_only(@salt1, @salt2, nhsnumber)
+        if !nhsnumber.to_s.empty? && !birthdate.to_s.empty? # && false to stop caching
+          @key_cache = {} if @key_cache.size > 1000 # Limit cache size
+          @key_cache[key] = [pseudo_id1, key_bundle, demog_key]
+        end
+      end
+      encrypted_demographics = NdrPseudonymise::SimplePseudonymisation.
+                               encrypt_data64(demog_key, all_demographics_hash.to_json)
+      packed_pseudoid_and_demographics = format('%s (%s) %s', pseudo_id1, key_bundle,
+                                                encrypted_demographics)
+      [[packed_pseudoid_and_demographics] + clinical_data(row)]
+    end
+
+    # Header row for CSV data
+    def csv_header_row
+      [PREAMBLE_V2_DEMOG_ONLY]
+    end
+
+    # Append the output of pseudonymise_row to a CSV file
+    def emit_csv_rows(out_csv, pseudonymised_row)
+      out_csv << pseudonymised_row[0]
+    end
+  end
+end

data/lib/ndr_pseudonymise/engine.rb

@@ -0,0 +1,6 @@
+module NdrPseudo
+  # base class for the engine
+  class Engine < ::Rails::Engine
+    config.eager_load_paths << File.expand_path('../../lib', __FILE__)
+  end
+end

data/lib/ndr_pseudonymise/ndr_encrypt/command_line.rb

@@ -0,0 +1,194 @@
+require 'optparse'
+
+module NdrPseudonymise
+  module NdrEncrypt
+    # ndr_encrypt command line utility entry points.
+    module CommandLine
+      # ndr_encrypt tool needs to support ruby 2.0
+      # rubocop:disable Layout/HeredocIndentation, Rails/Exit, Style/SlicingWithRange
+      USAGE = <<-USAGE.gsub(/^      /, '').freeze
+usage: ndr_encrypt [-v | --version] [-h | --help]
+                   <command> [<args>]
+
+These are common ndr_encrypt commands used in various situations:
+
+start a working area
+   init          Create an empty Git ndr_encrypt working copy
+
+work with files
+   add           Add file contents to the encrypted store and index
+   rm            TODO: Remove files from the encrypted store and index
+
+encryption key rotation and repository maintenance
+   gc            Cleanup unnecessary index entries and optimize the encrypted store
+   resilver      TODO
+   retire-key    TODO
+
+decrypt data
+   cat-files     TODO: Retrieve local file based on git_blobid
+   cat-remote    Retrieve remote file based on git_blobid
+   get           Retrieve local file(s) based on path in CSV index
+   TODO: decrypt single file without git_blobid
+
+Low-level Commands / Interrogators
+   cat-file      TODO: Provide content or type and size information for encrypted objects
+   ls-files      TODO: Show information about files in the index
+
+Low-level Commands / Manipulators
+   hash-object   TODO: Compute object ID and optionally create an encrypted object from a file
+      USAGE
+
+      COMMANDS = %w[init add cat-remote gc get].freeze
+
+      def self.run! # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        options = {}
+        parser = OptionParser.new do |opts|
+          # Hide TODOs in usage message
+          # rubocop:disable Style/SelectByRegexp
+          usage = USAGE.split("\n").reject { |s| s =~ /TODO/ }.join("\n")
+          # rubocop:enable Style/SelectByRegexp
+          opts.banner = "#{usage.chomp}\n\nAdditional options:"
+
+          opts.on('--base_url=URL', 'Remote repository URL') do |url|
+            options[:base_url] = url
+          end
+          opts.on('--key_name=NAME', /[A-Z0-9_-]*/i, 'Key name') do |name|
+            options[:key_name] = name
+          end
+          opts.on('--private_key=NAME', 'Private key filename') do |name|
+            options[:private_key] = name
+          end
+          opts.on('--pub_key=NAME', 'Public key filename') do |name|
+            options[:pub_key] = name
+          end
+          opts.on('--passin=OPTIONS', 'Pass in private key passphrase') do |name|
+            options[:passin] = name
+          end
+          opts.on('-p', 'Print downloaded object') do |v|
+            options[:pretty_print] = v
+          end
+        end
+
+        begin
+          parser.parse!
+        rescue OptionParser::ParseError => e
+          warn e
+          exit 1
+        end
+
+        parser.parse('--help') if ARGV.empty?
+        command = ARGV[0]
+        unless COMMANDS.include?(command)
+          warn <<-UNKNOWN_CMD
+ndr_encrypt: '#{command}' is not an ndr_encrypt command. See 'ndr_encrypt --help'.
+          UNKNOWN_CMD
+          exit 1
+        end
+        send(command.gsub('-', '_'), ARGV[1..-1], options)
+      end
+
+      def self.init(args, options) # rubocop:disable Metrics/MethodLength
+        required_options = %i[]
+        allowed_options = required_options
+        unless (options.keys - allowed_options).empty? &&
+               required_options.all? { |sym| options.key?(sym) } &&
+               args.size <= 1
+          warn <<-USAGE
+usage: ndr_encrypt init [<path>]
+          USAGE
+          exit 1
+        end
+
+        path = args[0] || Dir.pwd
+        action = if NdrEncrypt::Repository.new(repo_dir: path).init
+                   'Initialized empty'
+                 else
+                   'Reinitialized existing'
+                 end
+        $stdout.puts "#{action} ndr_encrypted encrypted store in " \
+                     "#{File.join(File.expand_path(path), NdrEncrypt::Repository::ENCRYPTED_DIR)}"
+      end
+
+      def self.add(args, options) # rubocop:disable Metrics/MethodLength
+        required_options = %i[key_name pub_key]
+        allowed_options = required_options
+        unless (options.keys - allowed_options).empty? &&
+               required_options.all? { |sym| options.key?(sym) }
+          warn <<-USAGE
+usage: ndr_encrypt add --key_name=<keyname> --pub_key=<path> [<path>...]
+          USAGE
+          exit 1
+        end
+        if args.empty?
+          warn 'Nothing specified, nothing added.'
+          return
+        end
+
+        path = Dir.pwd
+        repo = NdrEncrypt::Repository.new(repo_dir: path)
+        repo.add(args, key_name: options[:key_name], pub_key: options[:pub_key])
+      end
+
+      def self.cat_remote(args, options) # rubocop:disable Metrics/MethodLength
+        # TODO: Add -e option to check whether remote file exists
+        required_options = %i[key_name private_key base_url pretty_print]
+        allowed_options = required_options + %i[passin]
+        unless (options.keys - allowed_options).empty? &&
+               required_options.all? { |sym| options.key?(sym) } &&
+               args.size == 1
+          warn <<-USAGE
+usage: ndr_encrypt cat-remote --key_name=<keyname> --private_key=<path> --base_url=<url>
+                                -p <git_blobid>
+          USAGE
+          exit 1
+        end
+
+        git_blobid = args[0]
+        remote_store = NdrEncrypt::RemoteRepository.new(base_url: options[:base_url])
+        blob = remote_store.cat_remote(git_blobid, key_name: options[:key_name],
+                                                   private_key: options[:private_key],
+                                                   passin: options[:passin])
+        # TODO: Add error handling, for connection issues / file not found
+        $stdout.print blob
+      end
+
+      def self.gc(args, options)
+        required_options = %i[]
+        allowed_options = required_options
+        unless (options.keys - allowed_options).empty? &&
+               required_options.all? { |sym| options.key?(sym) } &&
+               args.empty?
+          warn <<-USAGE
+usage: ndr_encrypt gc
+          USAGE
+          exit 1
+        end
+
+        path = Dir.pwd
+        NdrEncrypt::Repository.new(repo_dir: path).gc(output_stream: $stdout)
+      end
+
+      def self.get(args, options) # rubocop:disable Metrics/MethodLength
+        required_options = %i[key_name private_key]
+        allowed_options = required_options + %i[passin]
+        unless (options.keys - allowed_options).empty? &&
+               required_options.all? { |sym| options.key?(sym) }
+          warn <<-USAGE
+usage: ndr_encrypt get --key_name=<keyname> --private_key=<path> [<path>...]
+          USAGE
+          exit 1
+        end
+        if args.empty?
+          warn 'Nothing specified, nothing to get.'
+          return
+        end
+
+        path = Dir.pwd
+        repo = NdrEncrypt::Repository.new(repo_dir: path)
+        repo.get(args, key_name: options[:key_name], private_key: options[:private_key],
+                       passin: options[:passin])
+      end
+      # rubocop:enable Layout/HeredocIndentation, Rails/Exit, Style/SlicingWithRange
+    end
+  end
+end