ndr_import 8.3.0 → 8.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 82e0bb47cb3a5c62b7cd5701c7196760cf9977357490a0e9ce708c2d814a51fc
-  data.tar.gz: 111f781ab628de952d6b305e4fd7f0f8eb412b6c7b7b539dca2786e46904b05d
+  metadata.gz: 1ff3d3bfb186b79ee65c5f10a9c67508bbd91668424197b7dad3f256fbc7c78b
+  data.tar.gz: 73a0245645a2996d35d0bc3eefcec7c6cdfa37808b9ffb4c82dd49cb70ecc2ab
 SHA512:
-  metadata.gz: f6d269d1d9e73ce154f7e8886a9109c16b74ba83d5b5c8f296f2c241609f1f5c9309afd08bee8538b34c36e534d48bacc5e1af6ff1fe89f89d9d30846c8841bc
-  data.tar.gz: e332a7c3af8857095e5f1dc0de7fd7693f7df4b639c3f1a34858b4f5c7af68b73dbff751142e485a0f836e7addf656043dd944816ad03ecf9b56797d8cd5a740
+  metadata.gz: 04d43b45585b8577b98b2b7c77295071a73db86dc1bdd0b213e6479ff8ed5027d7c6dc3248a04dc83d6398f17683e7f0421e5205f46e57d328b5efd57c427fc5
+  data.tar.gz: 3ac7c8948019c4d45ab0ae2b29e21fca1f21657e845252e659f437b8a81521f052a03aa320089ab1afa33dfdeee2c27d5f97766724c0453fcb15f50f77b52476

data/CHANGELOG.md

@@ -1,6 +1,9 @@
 ## [Unreleased]
 *no unreleased changes*
 
+## 8.4.0 / 2019-03-15
+* Added ability to extract and transform PDF form data (#24)
+
 ## 8.3.0 / 2019-03-04
 ### Added
 * Allow `klass` in the column level mapping to be embedded array.

data/README.md

@@ -2,7 +2,7 @@
 
 This is the Public Health England (PHE) National Disease Registers (NDR) Import ETL ruby gem, providing:
 
-1. file import handlers for *extracting* data from delimited files (csv, pipe, tab, thorn), .xls(x) spreadsheets, .doc(x) word documents, PDF, XML, 7-Zip and Zip files.
+1. file import handlers for *extracting* data from delimited files (csv, pipe, tab, thorn), .xls(x) spreadsheets, .doc(x) word documents, PDF, PDF AcroForms, XML, 7-Zip and Zip files.
 2. table mappers for *transforming* tabular and non-tabular data into key value pairs grouped by a common "klass".
 
 ## Installation

data/code_safety.yml

@@ -19,7 +19,7 @@ file safety:
   CHANGELOG.md:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 6271165699e2d886f94d45277baa955023e3f094
+    safe_revision: 9dd7962195b259862a3294aba3de061b391435eb
   CODE_OF_CONDUCT.md:
     comments: 
     reviewed_by: timgentry
@@ -39,7 +39,7 @@ file safety:
   README.md:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: f2feb0c430947839183d7223e60e7c29b2c0f846
+    safe_revision: 5cd2cd0b3a1e254d30d4acc28c6731825a1f84f5
   Rakefile:
     comments: 
     reviewed_by: josh.pencheon
@@ -59,15 +59,23 @@ file safety:
   lib/ndr_import.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: eca44583e9989159b45e90021dd1c65228447180
+    safe_revision: 24d6449fd0612552f132dfbf4cada2ae28d0469e
+  lib/ndr_import/acroform_reader.rb:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 24d6449fd0612552f132dfbf4cada2ae28d0469e
   lib/ndr_import/csv_library.rb:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: be12e57519d3737e8d3901d7b01485c6995708dd
+  lib/ndr_import/file/acro_form.rb:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 8a6ea666616c5b5d7c93cdf5aa019e8fc69d19e1
   lib/ndr_import/file/all.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: f2feb0c430947839183d7223e60e7c29b2c0f846
+    safe_revision: 5cd2cd0b3a1e254d30d4acc28c6731825a1f84f5
   lib/ndr_import/file/base.rb:
     comments: 
     reviewed_by: timgentry
@@ -180,6 +188,10 @@ file safety:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: ac30f66578ab380649be800a4426d917ddbcb329
+  lib/ndr_import/pdf_form/table.rb:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 5fd247eeb13a3f1356ab2d76ac9fabf9e19d5d36
   lib/ndr_import/standard_mappings.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -192,10 +204,14 @@ file safety:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: 3e3a852b58e8b169535e29029e535a10f6b9cd42
+  lib/ndr_import/unmapped_data_error.rb:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 5cd2cd0b3a1e254d30d4acc28c6731825a1f84f5
   lib/ndr_import/version.rb:
     comments: another check?
     reviewed_by: josh.pencheon
-    safe_revision: 6271165699e2d886f94d45277baa955023e3f094
+    safe_revision: 9dd7962195b259862a3294aba3de061b391435eb
   lib/ndr_import/xml/table.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -204,6 +220,10 @@ file safety:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: 607c0668f1fffd70d181bc1a31c4f56eed5f6189
+  test/file/acro_form_test.rb:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 5cd2cd0b3a1e254d30d4acc28c6731825a1f84f5
   test/file/base_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -227,7 +247,7 @@ file safety:
   test/file/registry_test.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: f2feb0c430947839183d7223e60e7c29b2c0f846
+    safe_revision: 5cd2cd0b3a1e254d30d4acc28c6731825a1f84f5
   test/file/seven_zip_test.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -292,10 +312,18 @@ file safety:
     comments: 
     reviewed_by: timgentry
     safe_revision: cf382902508a21a95b80ac4582fbbd117164e80e
+  test/pdf_form/table_test.rb:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 5cd2cd0b3a1e254d30d4acc28c6731825a1f84f5
   test/readme_test.rb:
     comments: 
     reviewed_by: timgentry
     safe_revision: cf382902508a21a95b80ac4582fbbd117164e80e
+  test/resources/acro_form.pdf:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 5cd2cd0b3a1e254d30d4acc28c6731825a1f84f5
   test/resources/blank_tab_test.xlsx:
     comments: reviewed contents
     reviewed_by: joshpencheon

data/lib/ndr_import/acroform_reader.rb

@@ -0,0 +1,33 @@
+require 'pdf-reader'
+
+module NdrImport
+  # PDF AcroForm reader using the pdf-reader gem
+  class AcroFormReader < ::PDF::Reader
+    def fields_hash
+      fields = {}
+      fields_from(acroform[:Fields]).each do |field|
+        field_name = field[:T]
+        unless field[:Subtype] == :Widget || field.key?(:Kids)
+          raise "Widgets or Radio boxes expected, found a #{field[:Subtype].inspect}"
+        end
+        raise "Non-unique column name #{field_name}" if fields.key?(field_name)
+        fields[field_name] = field[:V]
+      end
+      fields
+    end
+
+    private
+
+    def acroform
+      @objects.deref(root[:AcroForm])
+    end
+
+    def fields_from(refs)
+      Array(refs).flat_map do |ref|
+        value = @objects[ref]
+        # PDF has its own Hash class
+        value.is_a?(::Hash) ? value : fields_from(value)
+      end
+    end
+  end
+end

data/lib/ndr_import/file/acro_form.rb

@@ -0,0 +1,28 @@
+require 'pdf-reader'
+require 'ndr_support/safe_file'
+require_relative 'registry'
+
+module NdrImport
+  # This is one of a collection of file handlers that deal with individual formats of data.
+  # They can be instantiated directly or via the factory method Registry.tables
+  module File
+    # This class is an AcroForm PDF file handler that returns a single table
+    class AcroForm < Base
+      private
+
+      def rows(&block)
+        return enum_for(:rows) unless block
+
+        [reader.fields_hash].each(&block)
+      rescue NoMethodError
+        raise "Failed to read #{SafeFile.basename(@filename)} as an AcroForm PDF"
+      end
+
+      def reader
+        @reader ||= AcroFormReader.new(SafeFile.safepath_to_string(@filename))
+      end
+    end
+
+    Registry.register(AcroForm, 'acroform')
+  end
+end

data/lib/ndr_import/file/all.rb

@@ -1,4 +1,5 @@
 require_relative 'base'
+require_relative 'acro_form'
 require_relative 'delimited'
 require_relative 'docx'
 require_relative 'excel'

data/lib/ndr_import/pdf_form/table.rb

@@ -0,0 +1,71 @@
+require 'ndr_import/table'
+
+module NdrImport
+  module PdfForm
+    # This class maintains the state of a PDF form table mapping and encapsulates
+    # the logic required to transform a table of data into "records". Particular
+    # attention has been made to use enumerables throughout to help with the
+    # transformation of large quantities of data.
+    class Table < ::NdrImport::Table
+      def self.all_valid_options
+        super - %w[delimiter footer_lines format header_lines]
+      end
+
+      def footer_lines
+        0
+      end
+
+      def format
+        'acroform'
+      end
+
+      def header_lines
+        0
+      end
+
+      # This method transforms an incoming line (Hash) of data.
+      # Each of the klass masked mappings are applied to the hash values, which are reordered by
+      # the mappng definition, yielding the klass and fields for each mapped klass.
+      def transform_line(line, index)
+        return enum_for(:transform_line, line, index) unless block_given?
+
+        raise 'NdrImport::PdfForm::Table expects a Hash!' unless line.is_a? Hash
+
+        validate_column_mappings(line)
+
+        masked_mappings.each do |klass, klass_mappings|
+          ordered_line = order_values_by_mappings(line, klass_mappings)
+          fields       = mapped_line(ordered_line, klass_mappings)
+          next if fields[:skip].to_s == 'true'.freeze
+          yield(klass, fields, index)
+        end
+      end
+
+      private
+
+      # Ensure every key has a column mapping
+      def validate_column_mappings(line)
+        unmapped = []
+        line.each_key do |key|
+          next if column_names.include? key
+          unmapped << key
+        end
+        raise NdrImport::UnmappedDataError, unmapped if unmapped.any?
+      end
+
+      def column_name_from(column)
+        column[Strings::COLUMN] || column[Strings::STANDARD_MAPPING]
+      end
+
+      def column_names
+        @column_names ||= columns.map { |column| column_name_from(column) }
+      end
+
+      # Return an Array of the `hash` values in the order the columns are defined in the mapping,
+      # allowing mapped_line to work as normal
+      def order_values_by_mappings(hash, column_mappings)
+        column_mappings.map { |column_mapping| hash[column_name_from(column_mapping)].to_s }
+      end
+    end
+  end
+end

data/lib/ndr_import/unmapped_data_error.rb

@@ -0,0 +1,14 @@
+require 'active_support/core_ext/array/conversions'
+
+module NdrImport
+  # Raised if incoming data has not been mapped.
+  class UnmappedDataError < StandardError
+    attr_reader :keys
+
+    def initialize(keys)
+      @keys = keys
+      message = "Unmapped data: #{keys.to_sentence}"
+      super(message)
+    end
+  end
+end

data/lib/ndr_import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # This stores the current version of the NdrImport gem
 module NdrImport
-  VERSION = '8.3.0'.freeze
+  VERSION = '8.4.0'.freeze
 end

data/lib/ndr_import.rb

@@ -8,6 +8,9 @@ require 'ndr_import/table'
 require 'ndr_import/non_tabular/table'
 require 'ndr_import/fixed_width/table'
 require 'ndr_import/xml/table'
+require 'ndr_import/pdf_form/table'
+require 'ndr_import/unmapped_data_error'
+require 'ndr_import/acroform_reader'
 
 module NdrImport
   def self.root

data/test/file/acro_form_test.rb

@@ -0,0 +1,39 @@
+require 'test_helper'
+require 'ndr_import/file/acro_form'
+
+module NdrImport
+  module File
+    # Acro Form file handler tests
+    class AcroFormTest < ActiveSupport::TestCase
+      def setup
+        @permanent_test_files = SafePath.new('permanent_test_files')
+      end
+
+      test 'should read pdf correctly' do
+        file_path = @permanent_test_files.join('acro_form.pdf')
+        handler = NdrImport::File::AcroForm.new(file_path, nil)
+
+        expected_row = { 'Group3'                      => nil,
+                         'Textbox1_required'           => nil,
+                         'Textbox2_required'           => nil,
+                         'List Box_required'           => '3',
+                         'Dropdown_required'           => '3',
+                         'Textbox1_optional'           => nil,
+                         'Textbox2_optional'           => nil,
+                         'List Box_optional'           => '3',
+                         'Dropdown_optional'           => '3',
+                         'Date_required'               => nil,
+                         'Date__optional'              => nil,
+                         'Textbox3_numerical_required' => nil,
+                         'Textbox3_numerical_optional' => nil }
+
+        handler.tables.each do |tablename, sheet|
+          assert_nil tablename
+          assert_instance_of Enumerator, sheet
+          assert_equal 1, sheet.to_a.size
+          assert_equal expected_row, sheet.to_a.first
+        end
+      end
+    end
+  end
+end

data/test/file/registry_test.rb

@@ -12,7 +12,8 @@ module NdrImport
 
       test 'Registry.handlers' do
         assert_instance_of Hash, NdrImport::File::Registry.handlers
-        assert_equal %w[7z csv delimited doc docx nontabular pdf text txt xls xlsx xml_table zip],
+        assert_equal %w[7z acroform csv delimited doc docx nontabular
+                        pdf text txt xls xlsx xml_table zip],
                      NdrImport::File::Registry.handlers.keys.sort
       end
 

data/test/pdf_form/table_test.rb

@@ -0,0 +1,119 @@
+require 'test_helper'
+
+# This tests the NdrImport::PdfForm::Table mapping class
+module PdfForm
+  class TableTest < ActiveSupport::TestCase
+    def setup
+      @form_data = [{
+        'address2'        => 'Address 2',
+        'should_be_blank' => '',
+        'date_2'          => '2018-12-01',
+        'date_1'          => '2018-01-01',
+        'address1'        => 'Address'
+      }]
+    end
+
+    def test_should_transform_pdf_form_hash
+      table = NdrImport::PdfForm::Table.new(klass: 'SomeTestKlass',
+                                            columns: pdf_form_column_mapping)
+
+      assert_equal 'acroform', table.format
+
+      transformed_data = table.transform(@form_data)
+      assert_equal 1, transformed_data.count
+
+      expected_data = ['SomeTestKlass', { rawtext: {
+        'address1'          => 'Address',
+        'address2'          => 'Address 2',
+        'missing_from_data' => '',
+        'date_1'            => '2018-01-01',
+        'date_2'            => '2018-12-01',
+        'should_be_blank'   => ''
+      } }, 0]
+
+      klass, fields, index = *transformed_data.first
+      assert_equal expected_data[0],  klass
+      assert_equal expected_data[1],  fields
+      assert_equal expected_data[-1], index
+    end
+
+    def test_should_transform_mutli_klass_pdf_form_hash
+      table = NdrImport::PdfForm::Table.new(columns: multi_klass_pdf_form_column_mapping)
+
+      expected_data = [
+        ['TestKlassOne',
+         { rawtext:
+           { 'address1'          => 'Address',
+             'address2'          => 'Address 2',
+             'missing_from_data' => '' } },
+         0],
+        ['TestKlassTwo',
+         { rawtext:
+           { 'address1'        => 'Address',
+             'date_1'          => '2018-01-01',
+             'date_2'          => '2018-12-01',
+             'should_be_blank' => '' } },
+         0]
+      ]
+
+      transformed_data = table.transform(@form_data).to_a
+      assert_equal 2, transformed_data.count
+
+      expected_data.each_with_index do |expected, index|
+        transformed = transformed_data[index]
+        assert_equal expected, transformed
+      end
+    end
+
+    def test_should_fail_with_unmappped_form_data
+      table = NdrImport::PdfForm::Table.new(klass: 'SomeTestKlass',
+                                            columns: partial_pdf_form_column_mapping)
+
+      exception = assert_raises(NdrImport::UnmappedDataError) { table.transform(@form_data).to_a }
+      assert exception.message == 'Unmapped data: address2 and date_1'
+    end
+
+    def test_should_not_be_valid_with_bespoke_format
+      exception = assert_raises(ArgumentError) { NdrImport::PdfForm::Table.new(format: 'a_format') }
+      exception.message == 'Unrecognised options: ["format"]'
+    end
+
+    private
+
+    def pdf_form_column_mapping
+      [
+        { 'column' => 'address1' },
+        { 'column' => 'address2' },
+        { 'column' => 'missing_from_data' },
+        { 'column' => 'date_1' },
+        { 'column' => 'date_2' },
+        { 'column' => 'should_be_blank' }
+      ]
+    end
+
+    def multi_klass_pdf_form_column_mapping
+      [
+        { 'column' => 'address1',
+          'klass'  => %w[TestKlassOne TestKlassTwo] },
+        { 'column' => 'address2',
+          'klass'  => 'TestKlassOne' },
+        { 'column' => 'missing_from_data',
+          'klass'  => 'TestKlassOne' },
+        { 'column' => 'date_1',
+          'klass'  => 'TestKlassTwo' },
+        { 'column' => 'date_2',
+          'klass'  => 'TestKlassTwo' },
+        { 'column' => 'should_be_blank',
+          'klass'  => 'TestKlassTwo' }
+      ]
+    end
+
+    def partial_pdf_form_column_mapping
+      [
+        { 'column' => 'address1' },
+        { 'column' => 'date_2' },
+        { 'column' => 'should_be_blank' }
+      ]
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_import
 version: !ruby/object:Gem::Version
-  version: 8.3.0
+  version: 8.4.0
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-04 00:00:00.000000000 Z
+date: 2019-03-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -363,7 +363,9 @@ files:
 - gemfiles/Gemfile.rails51
 - gemfiles/Gemfile.rails52
 - lib/ndr_import.rb
+- lib/ndr_import/acroform_reader.rb
 - lib/ndr_import/csv_library.rb
+- lib/ndr_import/file/acro_form.rb
 - lib/ndr_import/file/all.rb
 - lib/ndr_import/file/base.rb
 - lib/ndr_import/file/delimited.rb
@@ -393,12 +395,15 @@ files:
 - lib/ndr_import/non_tabular/record.rb
 - lib/ndr_import/non_tabular/table.rb
 - lib/ndr_import/non_tabular_file_helper.rb
+- lib/ndr_import/pdf_form/table.rb
 - lib/ndr_import/standard_mappings.rb
 - lib/ndr_import/table.rb
 - lib/ndr_import/universal_importer_helper.rb
+- lib/ndr_import/unmapped_data_error.rb
 - lib/ndr_import/version.rb
 - lib/ndr_import/xml/table.rb
 - ndr_import.gemspec
+- test/file/acro_form_test.rb
 - test/file/base_test.rb
 - test/file/delimited_test.rb
 - test/file/docx_test.rb
@@ -421,7 +426,9 @@ files:
 - test/non_tabular/mapping_test.rb
 - test/non_tabular/table_test.rb
 - test/non_tabular_file_helper_test.rb
+- test/pdf_form/table_test.rb
 - test/readme_test.rb
+- test/resources/acro_form.pdf
 - test/resources/blank_tab_test.xlsx
 - test/resources/bomd.csv
 - test/resources/broken.csv
@@ -495,6 +502,7 @@ signing_key:
 specification_version: 4
 summary: NDR Import
 test_files:
+- test/file/acro_form_test.rb
 - test/file/base_test.rb
 - test/file/delimited_test.rb
 - test/file/docx_test.rb
@@ -517,7 +525,9 @@ test_files:
 - test/non_tabular/mapping_test.rb
 - test/non_tabular/table_test.rb
 - test/non_tabular_file_helper_test.rb
+- test/pdf_form/table_test.rb
 - test/readme_test.rb
+- test/resources/acro_form.pdf
 - test/resources/blank_tab_test.xlsx
 - test/resources/bomd.csv
 - test/resources/broken.csv