ndr_import 6.1.1 → 6.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0f55e93045c35c6bdbe7e87d04727c07ba4dc45aa780bbee7ce6715edf9dcfc1
-  data.tar.gz: 2418dd97d27989fa10e327e9b7111fdfcae08ff18bf05f83b2683e616e93487e
+  metadata.gz: 33380c2e5d1abc43cb56a040c8b6255ec6315bb9860506c173dcd573df777f11
+  data.tar.gz: fe9a51d153f9f64cdeaf872198f06887ba0c0becf9877e9a0f6cbb511160ae10
 SHA512:
-  metadata.gz: fdf1fba48a41e4e75f8a0d6cec5a08afc959a06bc887330b635ff801e33f3fbbd73632e1c4f24f5ea7c4e76c5499d7caa1470f908cc44ac8ba80e780cfaa2153
-  data.tar.gz: 8bcfc1bcb1fd6fc6848cb2362980d932d04a8cb67e5fd8bb05cbd36815d937f8fd06ec8bee14e200de1b7ca054958ecf13415fb49e6bbb304a1fd13c62056d1e
+  metadata.gz: e11257edc7d89f30f05943c9eb992f7c3660d66a15100092639840768843456e15a8b3babe16194f04387bd890f399fab58238842276e1a8c40209cfc127ddf4
+  data.tar.gz: 27f159c69d12780b967caa9c5223f8ab91a33fa6cd21b62ebaa3811ab7a8d115d388fc084a8e2501663cbb24d81a97ae4cf090bd377ae4ccd1bbdb617e0ea9ab

data/README.md

@@ -2,7 +2,7 @@
 
 This is the Public Health England (PHE) National Disease Registers (NDR) Import ETL ruby gem, providing:
 
-1. file import handlers for *extracting* data from delimited files (csv, pipe, tab, thorn), .xls(x) spreadsheets, .doc word documents, PDF, XML and Zip files.
+1. file import handlers for *extracting* data from delimited files (csv, pipe, tab, thorn), .xls(x) spreadsheets, .doc(x) word documents, PDF, XML and Zip files.
 2. table mappers for *transforming* tabular and non-tabular data into key value pairs grouped by a common "klass".
 
 ## Installation

data/code_safety.yml

@@ -35,7 +35,7 @@ file safety:
   README.md:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: ea7acbd1bbe99eb57f65c1dd381c309299ae3e08
+    safe_revision: 1b66cfcbb61dfac93c44889ca0ced5836101c20c
   Rakefile:
     comments: 
     reviewed_by: josh.pencheon
@@ -67,7 +67,7 @@ file safety:
   lib/ndr_import/file/all.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 2104514689a1a1286195fff18144a8cecb93048b
+    safe_revision: 1b66cfcbb61dfac93c44889ca0ced5836101c20c
   lib/ndr_import/file/base.rb:
     comments: 
     reviewed_by: timgentry
@@ -76,6 +76,10 @@ file safety:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: 902f5326d85372d9632de9869d6f56fc02b83a10
+  lib/ndr_import/file/docx.rb:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 1b66cfcbb61dfac93c44889ca0ced5836101c20c
   lib/ndr_import/file/excel.rb:
     comments: 
     reviewed_by: joshpencheon
@@ -159,7 +163,7 @@ file safety:
   lib/ndr_import/non_tabular/table.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: f7670cf27e137064c19490abc7df020fb6e95801
+    safe_revision: 337bf56e39f0f08cf7593b03867bb2da48630663
   lib/ndr_import/non_tabular_file_helper.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -179,11 +183,11 @@ file safety:
   lib/ndr_import/version.rb:
     comments: another check?
     reviewed_by: josh.pencheon
-    safe_revision: e350bd3f45bac82e5a05bd75eb87e9ae82486cf2
+    safe_revision: b85d90430543f238706d569bacd750ee50cb5493
   ndr_import.gemspec:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: a1bd9cb14dd5a660119a339795fb6a9bf0c1819e
+    safe_revision: 1b66cfcbb61dfac93c44889ca0ced5836101c20c
   test/file/base_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -192,6 +196,10 @@ file safety:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: 902f5326d85372d9632de9869d6f56fc02b83a10
+  test/file/docx_test.rb:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 1b66cfcbb61dfac93c44889ca0ced5836101c20c
   test/file/excel_test.rb:
     comments: 
     reviewed_by: joshpencheon
@@ -203,15 +211,15 @@ file safety:
   test/file/registry_test.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: dfe367e64c6e0ff80495cd7989fc50311d5b258f
+    safe_revision: 1b66cfcbb61dfac93c44889ca0ced5836101c20c
   test/file/text_test.rb:
     comments: 
     reviewed_by: timgentry
     safe_revision: 3107f711805b6d3b89d32ec923178425aa600dac
   test/file/word_test.rb:
     comments: 
-    reviewed_by: timgentry
-    safe_revision: c88000b32401b5ae9ef7f5878a9b630506ab5a94
+    reviewed_by: josh.pencheon
+    safe_revision: 1b66cfcbb61dfac93c44889ca0ced5836101c20c
   test/file/zip_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -255,7 +263,7 @@ file safety:
   test/non_tabular/table_test.rb:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: f7670cf27e137064c19490abc7df020fb6e95801
+    safe_revision: 337bf56e39f0f08cf7593b03867bb2da48630663
   test/non_tabular_file_helper_test.rb:
     comments: 
     reviewed_by: timgentry
@@ -312,6 +320,10 @@ file safety:
     comments: 
     reviewed_by: timgentry
     safe_revision: 45da71ebd3acbc0fe53755bcd75483ba17cb6924
+  test/resources/hello_world.docx:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 1b66cfcbb61dfac93c44889ca0ced5836101c20c
   test/resources/hello_world.pdf:
     comments: 
     reviewed_by: josh.pencheon
@@ -356,6 +368,10 @@ file safety:
     comments: 
     reviewed_by: timgentry
     safe_revision: 45da71ebd3acbc0fe53755bcd75483ba17cb6924
+  test/resources/not_a_word_file.docx:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: 1b66cfcbb61dfac93c44889ca0ced5836101c20c
   test/resources/not_sign_delimited.txt:
     comments: 
     reviewed_by: josh.pencheon

data/lib/ndr_import/file/all.rb

@@ -1,5 +1,6 @@
 require_relative 'base'
 require_relative 'delimited'
+require_relative 'docx'
 require_relative 'excel'
 require_relative 'pdf'
 require_relative 'text'

data/lib/ndr_import/file/docx.rb

@@ -0,0 +1,29 @@
+require 'docx'
+require 'ndr_support/safe_file'
+require_relative 'registry'
+
+module NdrImport
+  # This is one of a collection of file handlers that deal with individual formats of data.
+  # They can be instantiated directly or via the factory method Registry.tables
+  module File
+    # This class is a modern Word document file handler that returns a single table.
+    # It only works on .docx documents
+    class Docx < Base
+      private
+
+      def rows(&block)
+        return enum_for(:rows) unless block
+
+        doc = ::Docx::Document.open(SafeFile.safepath_to_string(@filename))
+
+        doc.paragraphs.each do |p|
+          yield(p.to_s)
+        end
+      rescue StandardError => e
+        raise("#{SafeFile.basename(@filename)} [#{e.class}: #{e.message}]")
+      end
+    end
+
+    Registry.register(Docx, 'docx')
+  end
+end

data/lib/ndr_import/non_tabular/table.rb

@@ -16,8 +16,9 @@ module NdrImport
 
       include UTF8Encoding
 
-      NON_TABULAR_OPTIONS = %w(capture_start_line start_line_pattern end_line_pattern remove_lines
-                               start_in_a_record end_in_a_record)
+      NON_TABULAR_OPTIONS = %w(capture_end_line capture_start_line start_line_pattern
+                               end_line_pattern remove_lines start_in_a_record
+                               end_in_a_record).freeze
 
       def self.all_valid_options
         super - %w(delimiter tablename_pattern header_lines footer_lines) + NON_TABULAR_OPTIONS
@@ -118,7 +119,7 @@ module NdrImport
             start_record(line)
           elsif line =~ @end_line_pattern
             # This is an end line
-            end_record
+            end_record(line)
           else
             @non_tabular_record << line if @in_a_record
           end
@@ -142,7 +143,9 @@ module NdrImport
 
       # Tabulate the record (if in one), flagged it as no longer being in a record
       # and set the record to be a new one.
-      def end_record
+      def end_record(line)
+        # Add the end line to the @non_tabular_record (if required) before ending the record
+        @non_tabular_record << line if @capture_end_line
         @tabular_array << @non_tabular_record.tabulate(column_mappings) if @in_a_record
         @in_a_record = false
         @non_tabular_record = NdrImport::NonTabular::Record.new

data/lib/ndr_import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # This stores the current version of the NdrImport gem
 module NdrImport
-  VERSION = '6.1.1'.freeze
+  VERSION = '6.2.0'.freeze
 end

data/ndr_import.gemspec

@@ -1,4 +1,3 @@
-# coding: utf-8
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'ndr_import/version'
@@ -27,11 +26,12 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'rubyzip', '~> 1.2', '>= 1.2.2'
   spec.add_dependency 'roo', '~> 2.0'
 
+  spec.add_dependency 'docx', '~> 0.3'
+  spec.add_dependency 'msworddoc-extractor', '0.2.0'
   spec.add_dependency 'nokogiri', '~> 1.8', '>= 1.8.2'
+  spec.add_dependency 'pdf-reader', '1.2.0' # Raises warnings on Ruby 2.4+
   spec.add_dependency 'roo-xls'
   spec.add_dependency 'spreadsheet', '1.0.3'
-  spec.add_dependency 'pdf-reader', '1.2.0' # Raises warnings on Ruby 2.4+
-  spec.add_dependency 'msworddoc-extractor', '0.2.0'
 
   spec.required_ruby_version = '>= 2.2'
 

data/test/file/docx_test.rb

@@ -0,0 +1,39 @@
+require 'test_helper'
+require 'ndr_import/file/docx'
+
+module NdrImport
+  module File
+    # Word .docx document file handler tests
+    class DocxTest < ActiveSupport::TestCase
+      def setup
+        @permanent_test_files = SafePath.new('permanent_test_files')
+      end
+
+      test 'should read word file' do
+        file_path = @permanent_test_files.join('hello_world.docx')
+        handler = NdrImport::File::Docx.new(file_path, nil)
+        handler.tables.each do |tablename, sheet|
+          assert_nil tablename
+          assert_instance_of Enumerator, sheet
+          assert_equal [
+            'Hello world, this is a modern word document',
+            'With more than one line of text',
+            'Three in fact'
+          ], sheet.to_a
+        end
+      end
+
+      test 'should raise exception on invalid word file' do
+        assert_raises RuntimeError do
+          file_path = @permanent_test_files.join('not_a_word_file.docx')
+          handler = NdrImport::File::Docx.new(file_path, nil)
+          handler.tables.each do |tablename, sheet|
+            assert_nil tablename
+            assert_instance_of Enumerator, sheet
+            sheet.to_a
+          end
+        end
+      end
+    end
+  end
+end

data/test/file/registry_test.rb

@@ -12,7 +12,7 @@ module NdrImport
 
       test 'Registry.handlers' do
         assert_instance_of Hash, NdrImport::File::Registry.handlers
-        assert_equal %w[csv delimited doc nontabular pdf text txt xls xlsx zip],
+        assert_equal %w[csv delimited doc docx nontabular pdf text txt xls xlsx zip],
                      NdrImport::File::Registry.handlers.keys.sort
       end
 

data/test/file/word_test.rb

@@ -1,5 +1,5 @@
 require 'test_helper'
-require 'ndr_import/file/pdf'
+require 'ndr_import/file/word'
 
 module NdrImport
   module File

data/test/non_tabular/table_test.rb

@@ -48,7 +48,7 @@ STR
 
   def test_all_valid_options
     valid_options = %w[
-      canonical_name capture_start_line columns end_in_a_record end_line_pattern
+      canonical_name capture_end_line capture_start_line columns end_in_a_record end_line_pattern
       filename_pattern format klass remove_lines start_in_a_record start_line_pattern
     ]
     assert_equal valid_options.sort,
@@ -322,6 +322,41 @@ STR
     refute results.any? { |result| result =~ /This is never captured/ }
   end
 
+  def test_should_capture_end_line
+    data = <<~STR.each_line
+111
+Lorem ipsum dolor sit amet.
+CAPTURE THIS CODE ABC
+111
+Lorem ipsum dolor sit amet.
+CAPTURE THIS CODE XYZ
+111
+Lorem ipsum dolor sit amet.
+CAPTURE THIS CODE 123
+STR
+
+    table = YAML.load <<-YML.strip_heredoc
+      --- !ruby/object:NdrImport::NonTabular::Table
+      start_line_pattern: !ruby/regexp /\\A111\\z/
+      end_line_pattern: !ruby/regexp /\\ACAPTURE THIS CODE/
+      capture_start_line: true
+      capture_end_line: true
+      klass: SomeTestKlass
+      columns:
+      - column: one
+        non_tabular_cell:
+          lines: -1
+          capture: !ruby/regexp /\\A(.*)\\z/i
+    YML
+    enum = table.transform(data)
+    assert_instance_of Enumerator, enum
+
+    results = enum.map { |_klass, fields, _index| fields[:rawtext]['one'] }
+
+    assert_equal 3, results.count
+    assert_equal 'CAPTURE THIS CODE ABC', results.first
+  end
+
   def test_should_capture
     table = YAML.load <<-YML.strip_heredoc
       --- !ruby/object:NdrImport::NonTabular::Table

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_import
 version: !ruby/object:Gem::Version
-  version: 6.1.1
+  version: 6.2.0
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-14 00:00:00.000000000 Z
+date: 2018-09-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -85,53 +85,53 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: docx
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.8.2
+        version: '0.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.8.2
+        version: '0.3'
 - !ruby/object:Gem::Dependency
-  name: roo-xls
+  name: msworddoc-extractor
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.2.0
 - !ruby/object:Gem::Dependency
-  name: spreadsheet
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.3
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.3
+        version: '1.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.8.2
 - !ruby/object:Gem::Dependency
   name: pdf-reader
   requirement: !ruby/object:Gem::Requirement
@@ -147,19 +147,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.2.0
 - !ruby/object:Gem::Dependency
-  name: msworddoc-extractor
+  name: roo-xls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: spreadsheet
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 1.0.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 1.0.3
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -331,6 +345,7 @@ files:
 - lib/ndr_import/file/all.rb
 - lib/ndr_import/file/base.rb
 - lib/ndr_import/file/delimited.rb
+- lib/ndr_import/file/docx.rb
 - lib/ndr_import/file/excel.rb
 - lib/ndr_import/file/pdf.rb
 - lib/ndr_import/file/registry.rb
@@ -360,6 +375,7 @@ files:
 - ndr_import.gemspec
 - test/file/base_test.rb
 - test/file/delimited_test.rb
+- test/file/docx_test.rb
 - test/file/excel_test.rb
 - test/file/pdf_test.rb
 - test/file/registry_test.rb
@@ -390,6 +406,7 @@ files:
 - test/resources/hello_utf8.txt
 - test/resources/hello_windows.txt
 - test/resources/hello_world.doc
+- test/resources/hello_world.docx
 - test/resources/hello_world.pdf
 - test/resources/hello_world.txt
 - test/resources/high_ascii_delimited.txt
@@ -401,6 +418,7 @@ files:
 - test/resources/normal_thorn.csv
 - test/resources/not_a_pdf.pdf
 - test/resources/not_a_word_file.doc
+- test/resources/not_a_word_file.docx
 - test/resources/not_sign_delimited.txt
 - test/resources/sample_xls.xls
 - test/resources/sample_xlsx.xlsx
@@ -447,6 +465,7 @@ summary: NDR Import
 test_files:
 - test/file/base_test.rb
 - test/file/delimited_test.rb
+- test/file/docx_test.rb
 - test/file/excel_test.rb
 - test/file/pdf_test.rb
 - test/file/registry_test.rb
@@ -477,6 +496,7 @@ test_files:
 - test/resources/hello_utf8.txt
 - test/resources/hello_windows.txt
 - test/resources/hello_world.doc
+- test/resources/hello_world.docx
 - test/resources/hello_world.pdf
 - test/resources/hello_world.txt
 - test/resources/high_ascii_delimited.txt
@@ -488,6 +508,7 @@ test_files:
 - test/resources/normal_thorn.csv
 - test/resources/not_a_pdf.pdf
 - test/resources/not_a_word_file.doc
+- test/resources/not_a_word_file.docx
 - test/resources/not_sign_delimited.txt
 - test/resources/sample_xls.xls
 - test/resources/sample_xlsx.xlsx