standard-procedure-consolidate 0.1.4 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f45c16dc8e6f37f91043673804db8076030a804ab6c08fa1139f4d86cdf54b79
-  data.tar.gz: 7c77802d6b578ac50b9f9c8b5bad9ff097918fab5644eb3c771b2baf98451a36
+  metadata.gz: 1bd2015cc0f88bc4b16bae7d8ecd335fb7b28c2bc940f81b1479e4a58ecce647
+  data.tar.gz: 44905254b9536f40f839d64d0448c4fb40b1a3720fb4d9ba3e2251468175eb9b
 SHA512:
-  metadata.gz: 2ed822b460867d75395fdb9253ef3501619781d3ab31acaca7db4daf699658f8253d4a063336bcc2321bf6dd08f0207e91600172f8ec84e4263af055ee9107a7
-  data.tar.gz: 37a8f85beb0b9c65ba76bf9457edcb9e141d1e87aca1f65c01b5a4e43f45f143c66695d5922eb3ffafca43625f9e3745db8fd2e3c516dd6ffb73172a8c9437ac
+  metadata.gz: 36dee64a5f21766e06e7b5eb213e5a5b4dee9ca04da8bde4b429c4d93c5bd49275226975c7e9887214ea5a3b5975c4f955b245ab45e0103656b928ab59854789
+  data.tar.gz: 687bba668eb1f137b67c82819561ec2e8c2c3bc9a54f03aa4b6ac76958ae3a15988af8db5f78e7e95adb59968b5764b32c37e71d257b7885d1eb0e10d6e3f56a

data/.devcontainer/devcontainer.json

@@ -1,7 +1,23 @@
 // For format details, see https://aka.ms/devcontainer.json. For config options, see the
 // README at: https://github.com/devcontainers/templates/tree/main/src/ruby
 {
-	"name": "Ruby",
-	"image": "mcr.microsoft.com/devcontainers/ruby:1-3.2-bullseye",
-	"postCreateCommand": "bundle install"
-}
+  "name": "Ruby",
+  "image": "mcr.microsoft.com/devcontainers/ruby:1-3.2-bullseye",
+  "postCreateCommand": "bundle install",
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "Shopify.ruby-extensions-pack",
+        "testdouble.vscode-standard-ruby",
+        "manuelpuyol.erb-linter",
+        "Shopify.ruby-lsp",
+        "aki77.rails-db-schema",
+        "miguel-savignano.ruby-symbols",
+        "sibiraj-s.vscode-scss-formatter",
+        "Thadeu.vscode-run-rspec-file",
+        "Cronos87.yaml-symbols",
+        "aliariff.vscode-erb-beautify"
+      ]
+    }
+  }
+}

data/.ruby-version

@@ -1 +1 @@
-3.2.2
+3.2.5

data/.standard.yml

@@ -1,3 +1,3 @@
 # For available configuration options, see:
 #   https://github.com/testdouble/standard
-ruby_version: 2.6
+ruby_version: 3.2.5

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## [0.3.0] - 2024-11-21
+
+Updated the code that examines the docx file for merge fields to deal with Word formatting tags being inserted in the middle of the merge fields.  
+
+## [0.2.0] - 2023-09-13
+
+Thrown away the mail-merge implementation and replaced it with a simple search/replace.  
+Added in the command line utilities for examining and consolidating documents.  
+
 ## [0.1.4] - 2023-09-11
 
 Updated which files get exclusions after crashes in production with some client files

data/README.md

@@ -1,51 +1,84 @@
 # Standard::Procedure::Consolidate
 
-A simple gem for performing mailmerge on Microsoft Word .docx files.
+A simple gem for performing search and replace on Microsoft Word .docx files.
 
-Important: I can't claim the credit for this - I found [this gist](https://gist.github.com/ericmason/7200448) and have just adapted it.
+Important: I can't claim the credit for this - I found [this gist](https://gist.github.com/ericmason/7200448) and have adapted it for my needs.
 
-It's pretty simple, so it probably won't work with complex Word documents, but it does what I need.  YMMV.
+## Search/Replace for field placeholders
 
+If you have a Word .docx file that looks like this (ignoring formatting): 
 
-## Installation
+```
+Dear {{ first_name }},
 
-    $ bundle add standard-procedure-consolidate
+Thank you for your purchase of {{ product }} on the {{ date }}.  We hope that with the proper care and attention it will give you years of happy use.  
+
+Regards
+
+{{ user_name }}
+```
+
+We have marked out the "fields" by using squiggly brackets - in a manner similar to (but simpler than) Liquid or Mustache templates.  In this example, we have fields for first_name, product, date and user_name.  
+
+Consolidate reads your .docx file, locates these fields and then replaces them with the values you have supplied, writing the output to a new file.  
+
+NOTE: These are not traditional Word "mail-merge fields" - these are just fragments of text that are within the Word document.  See the history section for why this does not work with merge-fields.  
 
 ## Usage
 
-To list the merge fields within a document:
+There is a command-line tool or a ruby library that you can include into your code.  
+
+Using the ruby library: 
 
 ```ruby
-Consolidate::Docx::Merge.open "/path/to/docx" do |merge|
-  merge.examine
-end and nil
+Consolidate::Docx::Merge.open "/path/to/file.docx" do |doc|
+  puts doc.field_names 
+
+  doc.data first_name: "Alice", product: "Palm Pilot", date: "23rd January 2002", user_name: "Bob"
+  doc.write_to "/path/to/merge-file.docx"
+end 
 ```
-To perform a merge, replacing merge fields with supplied values:
 
-```ruby
-Consolidate::Docx::Merge.open "/path/to/docx" do |merge|
-  merge.data "Name" => "Alice Aadvark", "Company" => "TinyCo", "Job_Title" => "CEO"
-  merge.write_to "/path/to/output.docx"
-end
+Using the command line: 
+
+```sh
+examine /path/to/file.docx 
+
+consolidate /path/to/file.docx /path/to/merge-file.docx first_name=Alice "product=Palm Pilot" "date=23rd January 2022" "user_name=Bob" 
+```
+
+If you want to see what the routine is doing you can add the `verbose` option.  
+
+```sh
+examine /path/to/file.docx verbose
+
+consolidate /path/to/file.docx /path/to/merge-file.docx first_name=Alice "product=Palm Pilot" "date=23rd January 2022" "user_name=Bob" verbose
 ```
 
-NOTE: The merge fields are case-sensitive - which is why they should be supplied as strings (using the older `{ "key" => "value" }` style ruby hash).
+### History
 
-If your merge isn't working, you can pass `verbose: true` to `open` and it will list the internal .xml documents it finds, the fields it finds within those .xml documents and the substitutions it is trying to perform.
+Originally, this gem was intended to open a Word .docx file, find the mailmerge fields within it and then substitute new values.  
 
-## How it works
+I managed to get a basic version of this working with a variety of different Word files and all seemed good.  Until my client reported that when they went to print the document, the mailmerge fields reappeared!  My best guess is that Word is thinking that print-time is a trigger for merging in a data source (for example, printing out a form letter to 200 customers), so all the substitution work that this gem does is then discarded and Word asks for the merge data again.  The frustrating thing is I can't figure out how Word keeps the references to the merge fields after they've been substituted.  
+
+So instead, this does a simple search and replace - looking for fields within squiggly brackets and substituting them.  
+
+### How it works
 
 This is a bit sketchy and pieced together from the [code I found]((https://gist.github.com/ericmason/7200448)) and various bits of skimming through the published file format.
 
-A .docx file (unlike the previous .doc file), is actually a .zip file that contains a number of .xml files.  The contents of the document are stored in these .xml files, along with configuration information and stuff like fonts and styles.
+A .docx file is actually a .zip file that contains a number of .xml documents.  Some of these are for storing formatting information (fonts, styles and various metadata) and some are for storing the actual document contents.  
+
+Consolidate looks for word/document.xml files, plus any files that match word/header*.xml, word/footer*.xml, word/footnotes*.xml and word/endnotes*.xml.  It parses the XML, looking for text nodes that contain squiggly brackets.  If it finds them, it then checks to see if we have supplied a data value for the matching field-name and replaces the contents of the node.  
 
-When setting up a merge field, Word adds some special tags into the .xml file.  There appear to be two different versions of how it does this - using `w:fldSimple` or `w:instrText` tags.  Consolidate goes through each .xml file within the document (ignoring some which are configuration related) and looks for these two types of tag.
+## Installation
+
+    $ bundle add standard-procedure-consolidate
 
-The `data` method uses the hash of `field: value` data you supply, copies the .xml files and performs a straight text substitution on any matching merge fields.  Then `write_to`
 
 ## Development
 
-The repo contains a .devcontainer folder - this contains instructions for a development container that has everything needed to build the project.  Once the container has started, you can use `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+The repo contains a .devcontainer folder - this contains instructions for a development container that has everything needed to build the project.  Once the container has started, you can use `bin/setup` to install dependencies. Then, run `bundle exec rake` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
 `bundle exec rake install` will install the gem on your local machine (obviously not from within the devcontainer though). To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 

data/Rakefile

@@ -7,4 +7,4 @@ RSpec::Core::RakeTask.new(:spec)
 
 require "standard/rake"
 
-task default: %i[spec standard]
+task default: %i[standard:fix spec]

data/checksums/standard-procedure-consolidate-0.2.0.gem.sha512

@@ -0,0 +1 @@
+872129b729c3531b72fbecc9bea75262ebc771d8e0a9f4d41b30ad87f73ee664aaccf8d602a410c3b500f067cd636f88c74eecb887cb68a345b6ea9d575c2140

data/checksums/standard-procedure-consolidate-0.3.0.gem.sha512

@@ -0,0 +1 @@
+098078b13e3bb4e3370cbc6bdb0195300489dfb33a5c27a201a4c5f1f0184bba73557264e674f49da41dbc62ad991e8ae99bef1debf417a5b1a363b1d56829f1

data/exe/consolidate

@@ -1,3 +1,34 @@
 #!/usr/bin/env ruby
 
 require "consolidate"
+
+if ARGV[0].nil? || ARGV[0] == ""
+  puts "# Standard::Procedure::Consolidate"
+  puts "## Mailmerge for simple Microsoft Word .docx files."
+  puts ""
+  puts "Create a new file with the mailmerge fields replaced by the data you have supplied"
+  puts "USAGE: consolidate path/to/myfile.docx path/to/mynewfile.docx \"field1=value1\" \"field2=value2\" \"field3=value3\""
+  puts ""
+else
+  input = ARGV[0]
+  output = ARGV[1]
+  data = {}
+  verbose = false
+  2.upto ARGV.length do |index|
+    arg = ARGV[index]
+    next if arg.nil?
+    if arg.strip == "verbose"
+      verbose = true
+    else
+      pieces = arg.split("=")
+      key = pieces.first.strip
+      value = pieces.last.strip
+      data[key] = value
+    end
+  end
+
+  Consolidate::Docx::Merge.open input, verbose: verbose do |doc|
+    doc.data data
+    doc.write_to output
+  end and nil
+end

data/exe/examine

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "consolidate"
+
+if ARGV[0].nil? || ARGV[0] == ""
+  puts "# Standard::Procedure::Consolidate"
+  puts "## Mailmerge for simple Microsoft Word .docx files."
+  puts ""
+  puts "Examine the mailmerge fields inside a .docx file"
+  puts "USAGE: examine path/to/myfile.docx"
+  puts "Option: examine path/to/myfile.docx verbose"
+  puts ""
+else
+  Consolidate::Docx::Merge.open ARGV[0], verbose: (ARGV[1] == "verbose") do |doc|
+    doc.examine
+  end and nil
+end

data/lib/consolidate/docx/merge.rb

@@ -11,24 +11,55 @@ module Consolidate
         path
       end
 
+      def initialize(path, verbose: false, &block)
+        @verbose = verbose
+        @output = {}
+        @zip = Zip::File.open(path)
+        @documents = load_documents
+        block&.call self
+      end
+
+      # Helper method to display the contents of the document and the merge fields from the CLI
       def examine
-        puts "Documents: #{extract_document_names}"
-        puts "Merge fields: #{extract_field_names}"
+        documents = document_names.join(", ")
+        fields = field_names.join(", ")
+        puts "Documents: #{documents}"
+        puts "Merge fields: #{fields}"
       end
 
-      def data fields = {}
-        fields = fields.transform_keys(&:to_s)
+      # Read all documents within the docx and extract any merge fields
+      def field_names
+        tag_nodes.collect do |tag_node|
+          field_name_from tag_node
+        end.compact.uniq
+      end
+
+      # List the documents stored within this docx
+      def document_names
+        @zip.entries.collect { |entry| entry.name }
+      end
+
+      # Substitute the data from the merge fields with the values provided
+      def data mapping = {}
+        mapping = mapping.transform_keys(&:to_s)
+
+        if verbose
+          puts "...substitutions..."
+          fields.each do |key, value|
+            puts "      #{key} => #{value}"
+          end
+        end
 
         @documents.each do |name, document|
-          result = document.dup
-          result = substitute_style_one_with result, fields
-          result = substitute_style_two_with result, fields
+          output_document = substitute document.dup, mapping: mapping, document_name: name
 
-          @output[name] = result.serialize save_with: 0
+          @output[name] = output_document.serialize save_with: 0
         end
       end
 
+      # Write the new document to the given path
       def write_to path
+        puts "...writing to #{path}" if verbose
         Zip::File.open(path, Zip::File::CREATE) do |out|
           zip.each do |entry|
             out.get_output_stream(entry.name) do |o|
@@ -38,7 +69,7 @@ module Consolidate
         end
       end
 
-      protected
+      private
 
       attr_reader :verbose
       attr_reader :zip
@@ -46,82 +77,62 @@ module Consolidate
       attr_reader :documents
       attr_accessor :output
 
-      def initialize(path, verbose: false, &block)
-        raise "No block given" unless block
-        @verbose = verbose
-        @output = {}
-        @documents = {}
-        begin
-          @zip = Zip::File.open(path)
-          @zip.entries.each do |entry|
-            next unless entry.name =~ /word\/(document|header|footer|footnotes|endnotes).?\.xml/
-            puts "Reading #{entry.name}" if verbose
-            xml = @zip.get_input_stream entry
-            @documents[entry.name] = Nokogiri::XML(xml) { |x| x.noent }
-          end
-          yield self
-        ensure
-          @zip.close
+      def load_documents
+        @zip.entries.each_with_object({}) do |entry, documents|
+          next unless entry.name.match?(/word\/(document|header|footer|footnotes|endnotes).?\.xml/)
+          puts "...reading #{entry.name}" if verbose
+          xml = @zip.get_input_stream entry
+          documents[entry.name] = Nokogiri::XML(xml) { |x| x.noent }
         end
+      ensure
+        @zip.close
       end
 
-      def extract_document_names
-        @zip.entries.collect { |entry| entry.name }.join(", ")
-      end
-
-      def extract_field_names
-        (extract_style_one + extract_style_two).uniq.join(", ")
-      end
-
-      def extract_style_one
+      # Collect all the nodes that contain merge fields
+      def tag_nodes
         documents.collect do |name, document|
-          (document / "//w:fldSimple").collect do |field|
-            value = field.attributes["instr"].value.strip
-            puts "...found #{value} (v1) in #{name}" if verbose
-            value.include?("MERGEFIELD") ? value.gsub("MERGEFIELD", "").strip : nil
-          end.compact
+          tag_nodes_for document
         end.flatten
       end
 
-      def extract_style_two
-        documents.collect do |name, document|
-          (document / "//w:instrText").collect do |instr|
-            value = instr.inner_text
-            puts "...found #{value} (v2) in #{name}" if verbose
-            value.include?("MERGEFIELD") ? value.gsub("MERGEFIELD", "").strip : nil
-          end.compact
-        end.flatten
+      # go through all w:t (Word Text???) nodes of the document
+      # find any nodes that contain "{{"
+      # then find the ancestor node that also includes the ending "}}"
+      # This collection of nodes contains all the merge fields for this document
+      def tag_nodes_for document
+        (document / "//w:t").collect do |node|
+          (node.children.any? { |child| child.content.include? "{{" }) ? enclosing_node_for_start_tag(node) : nil
+        end.compact
       end
 
-      def substitute_style_one_with document, fields
-        # Word's first way of doing things
-        (document / "//w:fldSimple").each do |field|
-          if field.attributes["instr"].value =~ /MERGEFIELD (\S+)/
-            text_node = (field / ".//w:t").first
-            next unless text_node
-            puts "...substituting v1 #{field.attributes["instr"]} with #{fields[$1]}" if verbose
-            text_node.inner_html = fields[$1].to_s
-          end
-        end
-        document
+      # Extract the merge field name from the node
+      def field_name_from(tag_node)
+        return nil unless (matches = tag_node.content.match(/{{\s*(\S+)\s*}}/))
+        field_name = matches[1].strip
+        puts "...field #{field_name} found in #{name}" if verbose
+        field_name.to_s
       end
 
-      def substitute_style_two_with document, fields
-        # Word's second way of doing things
-        (document / "//w:instrText").each do |instr|
-          if instr.inner_text =~ /MERGEFIELD (\S+)/
-            text_node = instr.parent.next_sibling.next_sibling.xpath(".//w:t").first
-            text_node ||= instr.parent.next_sibling.next_sibling.next_sibling.xpath(".//w:t").first
-            next unless text_node
-            puts "...substituting v2 #{instr.inner_text} with #{fields[$1]}" if verbose
-            text_node.inner_html = fields[$1].to_s
-          end
+      # Go through the given document, replacing any merge fields with the values provided
+      # and storing the results in a new document
+      def substitute document, document_name:, mapping: {}
+        tag_nodes_for(document).each do |tag_node|
+          field_name = field_name_from tag_node
+          next unless mapping.has_key? field_name
+          field_value = mapping[field_name]
+          puts "...substituting #{field_name} with #{field_value} in #{document_name}" if verbose
+          tag_node.content = tag_node.content.gsub(field_name, field_value).gsub(/{{\s*/, "").gsub(/\s*}}/, "")
+        rescue => ex
+          # Have to mangle the exception message otherwise it outputs the entire document
+          puts ex.message.to_s[0..255]
         end
         document
       end
 
-      def close
-        zip.close
+      # Find the ancestor node that contains both the start {{ text and the end }} text enclosing the merge field
+      def enclosing_node_for_start_tag(node)
+        return node if node.content.include? "}}"
+        node.parent.nil? ? nil : enclosing_node_for_start_tag(node.parent)
       end
     end
   end

data/lib/consolidate/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Consolidate
-  VERSION = "0.1.4"
+  VERSION = "0.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: standard-procedure-consolidate
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.3.0
 platform: ruby
 authors:
 - Rahoul Baruah
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-09-11 00:00:00.000000000 Z
+date: 2024-11-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rubyzip
@@ -43,11 +43,11 @@ email:
 - rahoulb@standardprocedure.app
 executables:
 - consolidate
+- examine
 extensions: []
 extra_rdoc_files: []
 files:
 - ".devcontainer/devcontainer.json"
-- ".nova/Configuration.json"
 - ".rspec"
 - ".ruby-version"
 - ".standard.yml"
@@ -60,7 +60,10 @@ files:
 - checksums/standard-procedure-consolidate-0.1.2.gem.sha512
 - checksums/standard-procedure-consolidate-0.1.3.gem.sha512
 - checksums/standard-procedure-consolidate-0.1.4.gem.sha512
+- checksums/standard-procedure-consolidate-0.2.0.gem.sha512
+- checksums/standard-procedure-consolidate-0.3.0.gem.sha512
 - exe/consolidate
+- exe/examine
 - lib/consolidate.rb
 - lib/consolidate/docx/merge.rb
 - lib/consolidate/version.rb
@@ -74,7 +77,7 @@ metadata:
   homepage_uri: https://github.com/standard-procedure/standard-procedure-consolidate
   source_code_uri: https://github.com/standard-procedure/standard-procedure-consolidate
   changelog_uri: https://github.com/standard-procedure/standard-procedure-consolidate/blob/main/CHANGELOG.md
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -89,8 +92,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
-signing_key:
+rubygems_version: 3.4.19
+signing_key: 
 specification_version: 4
 summary: Simple ruby mailmerge for Microsoft Word .docx files.
 test_files: []

data/.nova/Configuration.json

@@ -1,3 +0,0 @@
-{
-
-}