ndr_import 8.5.0 → 8.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9190c1ebeb14ff9e86f5fc64510f1bd4a0f6c28290727650b80b975cbac85624
-  data.tar.gz: f94476c5853d351c9e12542a00adba3ce708bf4ba7b37c16d1589ecd710a3342
+  metadata.gz: d8e9942cb20c52476a27af5b45a5bdb532171e7eb4cbf4b1d6cf40164623702a
+  data.tar.gz: 4e67817d307c143c1dbd3f3dbf17798e8864f8f246149b91d73099763d28488f
 SHA512:
-  metadata.gz: b8978a404f15132428414b8809b6fb7a98649fa44dde1c339b4df7894203913048a3aae6f69d5613779e5e26dcfb525315751c30d88c8a36870110e55ccd7ccb
-  data.tar.gz: dd3dcb71aa46792a49327b95b513f53279e74ae9b52cc2024802f9717d9149f31d1931bf707686840bc9f2301703c672bf14d02da737c39ed93333c8d23cff13
+  metadata.gz: 9b36561a8c48c9626dff51913c96426cfa10904f31ba8867056f77e5752583b83080b55b23b5f0b30e74c095e91e08d6563ca1f7bf7f7ae265d03501cf384cbc
+  data.tar.gz: a23881b758d08e34d0b82faa177a193402e50c0ea3373c575189f175af1bcd1e030ceeeacd3eb1a3e21b96b82c3aa8b5a1ee41d5f69aae53977609420b5cf139

data/CHANGELOG.md

@@ -1,6 +1,13 @@
 ## [Unreleased]
 *no unreleased changes*
 
+## 8.5.1 / 2019-05-15
+### Added
+* Add data loader tools (#39)
+
+### Fixed
+* Update Ruby/Rails supported versions. 2.5/5.2 is now minimum
+
 ## 8.5.0 / 2019-05-01
 ### Added
 * Allow encypted docx/xlsx files to be read using `:file_password` option (#37)

data/Gemfile

@@ -1,7 +1,4 @@
 source 'https://rubygems.org'
 
-# ooxml_decrypt is not published to rubygems.org so we need to specify where it is
-gem 'ooxml_decrypt', git: 'https://github.com/timgentry/ooxml_decrypt'
-
 # Specify your gem's dependencies in ndr_import.gemspec
 gemspec

data/README.md

@@ -52,6 +52,12 @@ See `test/readme_test.rb` for a more complete working example.
 
 More information on the workings of the mapper are available in the [wiki](https://github.com/PublicHealthEngland/ndr_import/wiki).
 
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
 ## Contributing
 
 1. Fork it ( https://github.com/PublicHealthEngland/ndr_import/fork )

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'bundler/setup'
+require 'ndr_import'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/code_safety.yml

@@ -15,11 +15,11 @@ file safety:
   ".travis.yml":
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
+    safe_revision: d3d9a987befeecb122a448d8d06e66d74da13fb5
   CHANGELOG.md:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: da7a850cb2abe53b7c0d4eaf4950634209999b67
+    safe_revision: 730152642d4d08454d8aa2f76abe2eda086c077c
   CODE_OF_CONDUCT.md:
     comments: 
     reviewed_by: timgentry
@@ -27,7 +27,7 @@ file safety:
   Gemfile:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
+    safe_revision: 13a1fae896c9d2b3fbe99a71d52045476f25b425
   Guardfile:
     comments: 
     reviewed_by: timgentry
@@ -39,23 +39,39 @@ file safety:
   README.md:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: 5cd2cd0b3a1e254d30d4acc28c6731825a1f84f5
+    safe_revision: e1d967c10059e8c635452838c3f3dd2b969d9ae4
   Rakefile:
     comments: 
     reviewed_by: josh.pencheon
     safe_revision: 5a7f26cecaabab20f4e666776f9166dcc3fa6bfe
-  gemfiles/Gemfile.rails50:
+  bin/console:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
-  gemfiles/Gemfile.rails51:
+    safe_revision: e1d967c10059e8c635452838c3f3dd2b969d9ae4
+  bin/setup:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
+    safe_revision: e1d967c10059e8c635452838c3f3dd2b969d9ae4
+  exe/pdf_acro_form_to_yaml:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: e1d967c10059e8c635452838c3f3dd2b969d9ae4
+  exe/pdf_to_text:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: e1d967c10059e8c635452838c3f3dd2b969d9ae4
+  exe/word_to_text:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: e1d967c10059e8c635452838c3f3dd2b969d9ae4
   gemfiles/Gemfile.rails52:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
+    safe_revision: 13a1fae896c9d2b3fbe99a71d52045476f25b425
+  gemfiles/Gemfile.rails60:
+    comments: 
+    reviewed_by: josh.pencheon
+    safe_revision: d3d9a987befeecb122a448d8d06e66d74da13fb5
   lib/ndr_import.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -215,7 +231,7 @@ file safety:
   lib/ndr_import/version.rb:
     comments: another check?
     reviewed_by: josh.pencheon
-    safe_revision: da7a850cb2abe53b7c0d4eaf4950634209999b67
+    safe_revision: 730152642d4d08454d8aa2f76abe2eda086c077c
   lib/ndr_import/xml/table.rb:
     comments: 
     reviewed_by: josh.pencheon
@@ -223,7 +239,7 @@ file safety:
   ndr_import.gemspec:
     comments: 
     reviewed_by: josh.pencheon
-    safe_revision: a69d4a57ddcf13cdc13c27bd2eb91a395fa7ea36
+    safe_revision: d3d9a987befeecb122a448d8d06e66d74da13fb5
   test/file/acro_form_test.rb:
     comments: 
     reviewed_by: josh.pencheon

data/exe/pdf_acro_form_to_yaml

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
+
+require 'ndr_import/acroform_reader'
+require 'optparse'
+require 'yaml'
+
+def program_name
+  File.basename(File.expand_path(__FILE__))
+end
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{program_name} [options] file"
+
+  opts.on('-h', '--help', 'Prints this help') do
+    puts opts
+    exit
+  end
+end.parse!
+
+reader = NdrImport::AcroFormReader.new(ARGV[0])
+
+puts reader.fields_hash.to_yaml

data/exe/pdf_to_text

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
+
+require 'pdf-reader'
+require 'optparse'
+
+def program_name
+  File.basename(File.expand_path(__FILE__))
+end
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{program_name} [options] file"
+
+  opts.on('-h', '--help', 'Prints this help') do
+    puts opts
+    exit
+  end
+end.parse!
+
+reader = PDF::Reader.new(ARGV[0])
+
+reader.pages.each do |page|
+  # written like this, and not file.write(page.text.split("\n")),
+  # to behave in exactly the same way as the importer
+  page.text.split("\n").each do |line|
+    puts line
+  end
+end

data/exe/word_to_text

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__) + '/../lib'))
+
+require 'msworddoc-extractor'
+require 'optparse'
+
+def program_name
+  File.basename(File.expand_path(__FILE__))
+end
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{program_name} [options] file"
+
+  opts.on('-h', '--help', 'Prints this help') do
+    puts opts
+    exit
+  end
+end.parse!
+
+filename = ARGV[0]
+extension = File.extname(filename)
+raise 'This text extractor currently only works for .doc files' unless extension == '.doc'
+
+doc = MSWordDoc::Extractor.load(filename)
+
+puts doc.whole_contents

data/gemfiles/Gemfile.rails52

@@ -1,8 +1,5 @@
 source 'https://rubygems.org'
 
-# ooxml_decrypt is not published to rubygems.org so we need to specify where it is
-gem 'ooxml_decrypt', git: 'https://github.com/timgentry/ooxml_decrypt'
-
 gemspec path: '..'
 
 gem 'activesupport', '~> 5.2.0'

data/gemfiles/Gemfile.rails60

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+gemspec path: '..'
+
+gem 'activesupport', '~> 6.0.0rc1'
+

data/lib/ndr_import/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 # This stores the current version of the NdrImport gem
 module NdrImport
-  VERSION = '8.5.0'.freeze
+  VERSION = '8.5.1'.freeze
 end

data/ndr_import.gemspec

@@ -12,16 +12,18 @@ Gem::Specification.new do |spec|
   spec.homepage      = 'https://github.com/PublicHealthEngland/ndr_import'
   spec.license       = 'MIT'
 
-  # Exclude older versions of this gem from the package.
-  spec.files         = `git ls-files -z`.split("\x0").reject { |s| s =~ %r{^pkg/} }
-  # SECURE BNS 2018-08-06: Minimise sharing of (public-key encrypted) slack secrets in .travis.yml
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
   spec.files         -= %w[.travis.yml] # Not needed in the gem
-  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
   spec.add_dependency 'activemodel'
-  spec.add_dependency 'activesupport', '~> 5.0'
+  spec.add_dependency 'activesupport', '>= 5.0', '< 7'
   spec.add_dependency 'ndr_support', '>= 5.3.2', '< 6'
 
   spec.add_dependency 'rubyzip', '~> 1.2', '>= 1.2.2'
@@ -36,7 +38,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'seven_zip_ruby', '~> 1.2'
   spec.add_dependency 'spreadsheet', '1.0.3'
 
-  spec.required_ruby_version = '>= 2.4'
+  spec.required_ruby_version = '>= 2.5'
 
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'rake', '~> 10.0'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ndr_import
 version: !ruby/object:Gem::Version
-  version: 8.5.0
+  version: 8.5.1
 platform: ruby
 authors:
 - NCRS Development Team
 autorequire: 
-bindir: bin
+bindir: exe
 cert_chain: []
-date: 2019-05-01 00:00:00.000000000 Z
+date: 2019-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activemodel
@@ -28,16 +28,22 @@ dependencies:
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '5.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '5.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
 - !ruby/object:Gem::Dependency
   name: ndr_support
   requirement: !ruby/object:Gem::Requirement
@@ -358,7 +364,10 @@ dependencies:
         version: '0'
 description: NDR ETL Importer
 email: []
-executables: []
+executables:
+- pdf_acro_form_to_yaml
+- pdf_to_text
+- word_to_text
 extensions: []
 extra_rdoc_files: []
 files:
@@ -372,10 +381,14 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- bin/console
+- bin/setup
 - code_safety.yml
-- gemfiles/Gemfile.rails50
-- gemfiles/Gemfile.rails51
+- exe/pdf_acro_form_to_yaml
+- exe/pdf_to_text
+- exe/word_to_text
 - gemfiles/Gemfile.rails52
+- gemfiles/Gemfile.rails60
 - lib/ndr_import.rb
 - lib/ndr_import/acroform_reader.rb
 - lib/ndr_import/csv_library.rb
@@ -418,83 +431,6 @@ files:
 - lib/ndr_import/version.rb
 - lib/ndr_import/xml/table.rb
 - ndr_import.gemspec
-- test/file/acro_form_test.rb
-- test/file/base_test.rb
-- test/file/delimited_test.rb
-- test/file/docx_test.rb
-- test/file/excel_test.rb
-- test/file/pdf_test.rb
-- test/file/registry_test.rb
-- test/file/seven_zip_test.rb
-- test/file/text_test.rb
-- test/file/word_test.rb
-- test/file/xml_test.rb
-- test/file/zip_test.rb
-- test/fixed_width/table_test.rb
-- test/helpers/file/delimited_test.rb
-- test/helpers/file/excel_test.rb
-- test/helpers/file/pdf_test.rb
-- test/helpers/file/word_test.rb
-- test/helpers/file/xml_test.rb
-- test/helpers/file/zip_test.rb
-- test/mapper_test.rb
-- test/non_tabular/mapping_test.rb
-- test/non_tabular/table_test.rb
-- test/non_tabular_file_helper_test.rb
-- test/pdf_form/table_test.rb
-- test/readme_test.rb
-- test/resources/acro_form.pdf
-- test/resources/blank_tab_test.xlsx
-- test/resources/bomd.csv
-- test/resources/broken.csv
-- test/resources/filesystem_paths.yml
-- test/resources/flat_file.pdf
-- test/resources/flat_file.txt
-- test/resources/flat_file.yml
-- test/resources/hello_utf16be.txt
-- test/resources/hello_utf16le.txt
-- test/resources/hello_utf8.txt
-- test/resources/hello_windows.txt
-- test/resources/hello_world.doc
-- test/resources/hello_world.docx
-- test/resources/hello_world.pdf
-- test/resources/hello_world.txt
-- test/resources/high_ascii_delimited.txt
-- test/resources/high_ascii_delimited_example_two.txt
-- test/resources/malformed.csv
-- test/resources/malformed.xml
-- test/resources/malformed_pipe.csv
-- test/resources/normal.7z
-- test/resources/normal.csv
-- test/resources/normal.csv.zip
-- test/resources/normal_pipe.csv
-- test/resources/normal_thorn.csv
-- test/resources/not_a_pdf.pdf
-- test/resources/not_a_word_file.doc
-- test/resources/not_a_word_file.docx
-- test/resources/not_sign_delimited.txt
-- test/resources/password_protected_hello_world.docx
-- test/resources/password_protected_sample_xlsx.xlsx
-- test/resources/sample.xml
-- test/resources/sample_xls.xls
-- test/resources/sample_xlsx.xlsx
-- test/resources/sheet_streaming.xls
-- test/resources/sheet_streaming.xlsx
-- test/resources/standard_mappings.yml
-- test/resources/txt_file_xls_extension.xls
-- test/resources/txt_file_xlsx_extension.xlsx
-- test/resources/utf-16be_xml.xml
-- test/resources/utf-16be_xml_with_declaration.xml
-- test/resources/utf-16le_xml.xml
-- test/resources/utf-8_xml.xml
-- test/resources/windows-1252_xml.xml
-- test/resources/windows.csv
-- test/resources/xlsx_file_xls_extension.xls
-- test/standard_mappings_test.rb
-- test/table_test.rb
-- test/test_helper.rb
-- test/universal_importer_helper_test.rb
-- test/xml/table_test.rb
 homepage: https://github.com/PublicHealthEngland/ndr_import
 licenses:
 - MIT
@@ -507,7 +443,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.4'
+      version: '2.5'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -518,81 +454,4 @@ rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: NDR Import
-test_files:
-- test/file/acro_form_test.rb
-- test/file/base_test.rb
-- test/file/delimited_test.rb
-- test/file/docx_test.rb
-- test/file/excel_test.rb
-- test/file/pdf_test.rb
-- test/file/registry_test.rb
-- test/file/seven_zip_test.rb
-- test/file/text_test.rb
-- test/file/word_test.rb
-- test/file/xml_test.rb
-- test/file/zip_test.rb
-- test/fixed_width/table_test.rb
-- test/helpers/file/delimited_test.rb
-- test/helpers/file/excel_test.rb
-- test/helpers/file/pdf_test.rb
-- test/helpers/file/word_test.rb
-- test/helpers/file/xml_test.rb
-- test/helpers/file/zip_test.rb
-- test/mapper_test.rb
-- test/non_tabular/mapping_test.rb
-- test/non_tabular/table_test.rb
-- test/non_tabular_file_helper_test.rb
-- test/pdf_form/table_test.rb
-- test/readme_test.rb
-- test/resources/acro_form.pdf
-- test/resources/blank_tab_test.xlsx
-- test/resources/bomd.csv
-- test/resources/broken.csv
-- test/resources/filesystem_paths.yml
-- test/resources/flat_file.pdf
-- test/resources/flat_file.txt
-- test/resources/flat_file.yml
-- test/resources/hello_utf16be.txt
-- test/resources/hello_utf16le.txt
-- test/resources/hello_utf8.txt
-- test/resources/hello_windows.txt
-- test/resources/hello_world.doc
-- test/resources/hello_world.docx
-- test/resources/hello_world.pdf
-- test/resources/hello_world.txt
-- test/resources/high_ascii_delimited.txt
-- test/resources/high_ascii_delimited_example_two.txt
-- test/resources/malformed.csv
-- test/resources/malformed.xml
-- test/resources/malformed_pipe.csv
-- test/resources/normal.7z
-- test/resources/normal.csv
-- test/resources/normal.csv.zip
-- test/resources/normal_pipe.csv
-- test/resources/normal_thorn.csv
-- test/resources/not_a_pdf.pdf
-- test/resources/not_a_word_file.doc
-- test/resources/not_a_word_file.docx
-- test/resources/not_sign_delimited.txt
-- test/resources/password_protected_hello_world.docx
-- test/resources/password_protected_sample_xlsx.xlsx
-- test/resources/sample.xml
-- test/resources/sample_xls.xls
-- test/resources/sample_xlsx.xlsx
-- test/resources/sheet_streaming.xls
-- test/resources/sheet_streaming.xlsx
-- test/resources/standard_mappings.yml
-- test/resources/txt_file_xls_extension.xls
-- test/resources/txt_file_xlsx_extension.xlsx
-- test/resources/utf-16be_xml.xml
-- test/resources/utf-16be_xml_with_declaration.xml
-- test/resources/utf-16le_xml.xml
-- test/resources/utf-8_xml.xml
-- test/resources/windows-1252_xml.xml
-- test/resources/windows.csv
-- test/resources/xlsx_file_xls_extension.xls
-- test/standard_mappings_test.rb
-- test/table_test.rb
-- test/test_helper.rb
-- test/universal_importer_helper_test.rb
-- test/xml/table_test.rb
+test_files: []