ndr_pseudonymise 0.4.1

data/lib/ndr_pseudonymise.rb

@@ -0,0 +1,16 @@
+require 'active_support'
+
+require 'ndr_pseudonymise/version'
+require 'ndr_pseudonymise/pseudonymisation_specification'
+
+require 'ndr_pseudonymise/client'
+
+require 'ndr_pseudonymise/demographics_only_pseudonymiser'
+require 'ndr_pseudonymise/prescription_pseudonymiser'
+require 'ndr_pseudonymise/progress_printer'
+require 'ndr_pseudonymise/pseudonymisation_specification'
+require 'ndr_pseudonymise/simple_pseudonymisation'
+
+# Pseudonymise CSV data for matching purposes
+module NdrPseudonymise
+end

data/lib/rsa_aes_cbc.rb

@@ -0,0 +1,114 @@
+# From https://gist.github.com/Zapotek/981959/raw/d7353edad1bd88110dc1f03dcc47257219d1624e/rsa_aes_cbc.rb
+# To generate a public / private key pair:
+# require 'openssl'
+# new_key = OpenSSL::PKey::RSA.generate( 15360 )  # 4096 probably OK
+# File.open("./new_public.pem", "w") { |f| f.puts new_key.public_key }
+# File.open("./new_private.pem", "w") { |f| f.puts new_key.to_pem }
+
+require 'openssl'
+require 'yaml'
+require 'base64'
+
+# SECURE: TVB Mon 14 Oct 2013 13:33:06 BST
+# All the encryption keys are generated by standard libraries.
+# The data is encrypted with AES symmetric key. The symmetric key
+# is encrypted with RSA.
+#
+
+# KH: 20160914 - changed:
+#
+#   Base64.encode64 to Base64.strict_encode64 in encrypt()
+#
+# to remove newlines, to be consistent with the rest of the ndr_pseudonymise code.
+#   Base64.decode64 is kept in decrypt(), as this (correctly) ignores newlines produced
+#   by existing encrypted data using encrypt().
+#
+# http://ruby-doc.org/stdlib-2.3.1/libdoc/base64/rdoc/Base64.html
+# https://tools.ietf.org/html/rfc4648#section-3.1
+
+
+
+# Simple hybrid crypto class using RSA for public key encryption and AES with CBC
+# for bulk data encryption/decryption.
+#
+# RSA is used to encrypt the AES primitives which are used to encrypt the plaintext.
+#
+# @author: Tasos "Zapotek" Laskos
+#                                      <tasos.laskos@gmail.com>
+#                                      <zapotek@segfault.gr>
+# @version: 0.1
+class RSA_AES_CBC
+  #
+  # If only encryption is required the private key parameter can be omitted.
+  #
+  # @param  [String]  public_pem   location of the Public key in PEM format
+  # @param  [String]  private_pem  location of the Private key in PEM format
+  #
+  def initialize(public_pem, private_pem = nil)
+    @public_pem = public_pem
+    @private_pem = private_pem
+  end
+
+  #
+  # Encrypts data and returns a Base64 representation of the ciphertext
+  # and AES CBC primitives encrypted using the public key.
+  #
+  # @param  [String]  data
+  #
+  # @return  [String]   Base64 representation of the ciphertext
+  #                       and AES CBC primitives encrypted using the public key.
+  #
+  def encrypt(data)
+    rsa = OpenSSL::PKey::RSA.new(@public_pem)
+
+    # encrypt with 256 bit AES with CBC
+    aes = OpenSSL::Cipher.new('aes-256-cbc')
+    aes.encrypt
+
+    # use random key and IV
+    aes.key = key = aes.random_key
+    aes.iv  = iv  = aes.random_iv
+
+    # this will hold all primitives and ciphertext
+    primitives = {}
+
+    primitives['ciphertext'] = aes.update(data)
+    primitives['ciphertext'] << aes.final
+
+    primitives['key'] = rsa.public_encrypt(key)
+    primitives['iv']  = rsa.public_encrypt(iv)
+
+    # serialize everything and base64 encode it
+    Base64.strict_encode64(primitives.to_yaml)
+  end
+
+  #
+  # Decrypts data.
+  #
+  # @param  [String]  data
+  #
+  # @return  [String]   plaintext
+  #
+  def decrypt(data)
+    rsa = OpenSSL::PKey::RSA.new(@private_pem)
+
+    # decrypt with 256 bit AES with CBC
+    aes = OpenSSL::Cipher.new('aes-256-cbc')
+    aes.decrypt
+
+    # unencode and unserialize to get the primitives and ciphertext
+    primitives = YAML.load(Base64.decode64(data))
+
+    aes.key = rsa.private_decrypt(primitives['key'])
+    aes.iv  = rsa.private_decrypt(primitives['iv'])
+
+    plaintext = aes.update(primitives['ciphertext'])
+    plaintext << aes.final
+
+    plaintext
+  end
+end
+
+# crypto = RSA_AES_CBC.new( 'public.pem', 'private.pem' )
+# ciphered = crypto.encrypt( 'Foo' )
+# puts crypto.decrypt( ciphered )

data/ndr_pseudonymise.gemspec

@@ -0,0 +1,36 @@
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'ndr_pseudonymise/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'ndr_pseudonymise'
+  spec.version       = NdrPseudonymise::VERSION
+  spec.authors       = ['NCRS development team']
+  spec.email         = []
+
+  spec.summary       = 'Provide pseudonymisation facilities.'
+  spec.description   = 'Provide pseudonymisation facilities.'
+  spec.homepage      = 'https://github.com/NHSDigital/ndr_pseudonymise'
+  spec.license       = 'MIT'
+
+  ignore_files_re    = %r{^(\.github|test|spec|features|gemfiles|)/|.travis.yml|code_safety.yml}
+  spec.files         = `git ls-files -z`.split("\x0").
+                       reject { |f| f.match(ignore_files_re) }
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  # Object methods like .blank?
+  spec.add_dependency 'activesupport'
+
+  spec.add_development_dependency 'activesupport'
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'minitest', '>= 5.0'
+  spec.add_development_dependency 'mocha'
+  spec.add_development_dependency 'ndr_dev_support', '>= 6.0'
+  spec.add_development_dependency 'ndr_import'
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'rake', '>= 12.3.3'
+
+  spec.required_ruby_version = '>= 2.6.0'
+end

data/script/ndr_encrypt/README.md

@@ -0,0 +1,154 @@
+# ndr_encrypt script
+
+`ndr_encrypt` encrypts images and other files, allowing them to be hosted and
+retrieved safely. The files still need to be hosted securely, but public /
+private key encryption provides additional protection.
+
+## Overview
+
+We define a simple suite of tools, `ndr_encrypt`, to generate the encrypted
+image files, and the data that lets us access them. These work in a similar way
+to git object storage, and require minimal software to run in a standard Linux
+/ macOS environment. (Related reading:
+https://git-scm.com/book/en/v2/Git-Internals-Git-Objects)
+
+With these tools, you can:
+1. transform a nested directory tree of files into an encrypted storage
+   representation + CSV file suitable for import to a SQL database
+2. identify and decrypt an image, using an entry from the CSV file
+3. recover the original contents of an unknown encrypted file (but not the
+   original filename), and use the CSV file to identify the original file
+   [TODO: not yet implemented]
+4. rewrite the encrypted files using a new encryption key [TODO: not yet
+   implemented]
+
+## Usage
+
+```
+usage: ndr_encrypt [-v | --version] [-h | --help]
+                   <command> [<args>]
+
+These are common ndr_encrypt commands used in various situations:
+
+start a working area
+   init          Create an empty Git ndr_encrypt working copy
+
+work with files
+   add           Add file contents to the encrypted store and index
+
+encryption key rotation and repository maintenance
+   gc            Cleanup unnecessary index entries and optimize the encrypted store
+
+decrypt data
+   cat-remote    Retrieve remote file based on git_blobid
+   get           Retrieve local file(s) based on path in CSV index
+
+Low-level Commands / Interrogators
+
+Low-level Commands / Manipulators
+
+Additional options:
+        --base_url=URL               Remote repository URL
+        --key_name=NAME              Key name
+        --private_key=NAME           Private key filename
+        --pub_key=NAME               Public key filename
+        --passin=OPTIONS             Pass in private key passphrase
+    -p                               Print downloaded object
+```
+
+`ndr_encrypt` requires ruby 2.0 or later to be installed
+
+## Simple Usage Example
+
+``` shell
+# Set up an image repository:
+ndr_encrypt init images
+cd images
+
+# Set up encryption / decryption keys:
+# Use a strong passphrase, e.g. by running openssl rand -hex 32
+echo Use a strong passphrase, e.g. `openssl rand -hex 32`
+keyname=ourkey1
+openssl genpkey -algorithm RSA -out ourkey1.pem -aes-256-cbc -pkeyopt rsa_keygen_bits:4096
+openssl rsa -in ourkey1.pem -out ourkey1.pub -outform PEM -pubout
+
+# Create a sample .gif file "test/dir/ok.gif"
+mkdir -p test/dir
+base64 --decode > test/dir/ok.gif <<BASE64
+R0lGODlhDAAIAPABAAAAAP///yH5BAAAAAAAIf8LSW1hZ2VNYWdpY2sOZ2Ft
+bWE9MC40NTQ1NDUALAAAAAAMAAgAAAITjI8HC9GuTJvozRchVQz6BIZgAQA7
+BASE64
+
+# Add the object to the repository
+ndr_encrypt add --key_name=ourkey1 --pub_key=ourkey1.pub test/dir/ok.gif
+
+# Move aside the original file, to test recovery
+mv test/dir/ok.gif{,.orig}
+
+# Recover file from the repository, prompting for the passphrase
+# (This uses the CSV index file ndr_encrypted/index.csv and
+#  the encrypted object store in ndr_encrypted/objects)
+ndr_encrypt get --key_name=ourkey1 --private_key=ourkey1.pem test/dir/ok.gif
+
+# Ensure the recovered file is identical
+diff -s test/dir/ok.gif{.orig,}
+
+# Check index contents (for hashes used in next example)
+cat ndr_encrypted/index.csv
+```
+
+## Retrieving files hosted on a webserver
+
+To retrieve files from a webserver, we assume that the contents of the index
+file `ndr_encrypted/index.csv` has been moved to a table, and the object store
+contents of `ndr_encrypted/objects/` have been hosted on a webserver or S3
+buckets, e.g. inside `https://example.org/encrypted/storage/`
+
+### Using `ndr_pseudonymise` gem
+
+``` ruby
+require 'ndr_pseudonymise/ndr_encrypt'
+
+key_name = 'ourkey1'
+private_key = 'ourkey1.pem'
+private_passphrase = begin # Should be read from encrypted credential storage
+  require 'io/console'
+  IO::console.getpass("Enter decryption passphrase for #{private_key.inspect}: ")
+end
+
+base_url = 'https://example.org/encrypted/storage/'
+git_blobid = 'f29bddf64c444f663d106568f4a81a22151ed3f97b0ec0c2a5ab25a0e8a02515'
+
+remote_repo = NdrPseudonymise::NdrEncrypt::RemoteRepository.new(base_url: base_url)
+decrypted_data = remote_repo.cat_remote(
+  git_blobid, key_name: key_name, private_key: private_key,
+              passin: "pass:#{private_passphrase}"
+)
+```
+
+### Using `ndr_encrypt` command line inside a ruby applcation
+
+``` ruby
+require 'open3'
+
+key_name = 'ourkey1'
+private_key = 'ourkey1.pem'
+private_passphrase = begin # Should be read from encrypted credential storage
+  require 'io/console'
+  IO::console.getpass("Enter decryption passphrase for #{private_key.inspect}: ")
+end
+
+base_url = 'https://example.org/encrypted/storage/'
+git_blobid = 'f29bddf64c444f663d106568f4a81a22151ed3f97b0ec0c2a5ab25a0e8a02515'
+
+decrypted_data = Open3.capture2(
+                  'ndr_encrypt', 'cat-remote', '-p', "--key_name=#{key_name}",
+                  "--private_key=#{private_key}", "--base_url=#{base_url}",
+                  '--passin=stdin', git_blobid,
+                  stdin_data: private_passphrase, binmode: true
+                 )[0]
+```
+
+## Low-level object manipulation
+
+TODO

data/script/ndr_encrypt/ndr_encrypt

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift(File.join(__dir__, '../../lib'))
+require 'ndr_pseudonymise/ndr_encrypt'
+NdrPseudonymise::NdrEncrypt::CommandLine.run!

metadata

@@ -0,0 +1,197 @@
+--- !ruby/object:Gem::Specification
+name: ndr_pseudonymise
+version: !ruby/object:Gem::Version
+  version: 0.4.1
+platform: ruby
+authors:
+- NCRS development team
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2022-10-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ndr_dev_support
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.0'
+- !ruby/object:Gem::Dependency
+  name: ndr_import
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 12.3.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 12.3.3
+description: Provide pseudonymisation facilities.
+email: []
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rubocop.yml"
+- CHANGELOG.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/ndr_pseudonymise.rb
+- lib/ndr_pseudonymise/client.rb
+- lib/ndr_pseudonymise/demographics_only_pseudonymiser.rb
+- lib/ndr_pseudonymise/engine.rb
+- lib/ndr_pseudonymise/ndr_encrypt.rb
+- lib/ndr_pseudonymise/ndr_encrypt/command_line.rb
+- lib/ndr_pseudonymise/ndr_encrypt/encrypted_object.rb
+- lib/ndr_pseudonymise/ndr_encrypt/remote_repository.rb
+- lib/ndr_pseudonymise/ndr_encrypt/repository.rb
+- lib/ndr_pseudonymise/prescription_pseudonymiser.rb
+- lib/ndr_pseudonymise/progress_printer.rb
+- lib/ndr_pseudonymise/pseudonymisation_specification.rb
+- lib/ndr_pseudonymise/pseudonymised_file_converter.rb
+- lib/ndr_pseudonymise/pseudonymised_file_wrapper.rb
+- lib/ndr_pseudonymise/simple_pseudonymisation.rb
+- lib/ndr_pseudonymise/version.rb
+- lib/rsa_aes_cbc.rb
+- ndr_pseudonymise.gemspec
+- script/ndr_encrypt/README.md
+- script/ndr_encrypt/ndr_encrypt
+homepage: https://github.com/NHSDigital/ndr_pseudonymise
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.33
+signing_key:
+specification_version: 4
+summary: Provide pseudonymisation facilities.
+test_files: []