neu-mods 0.2.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fb4c30744c189d3e0b4bfc0f84bb75175685bcb142dd6dc35dc55cfbcbdaf0a2
-  data.tar.gz: a5cd73f764edf2da97e05787788124bed168c9dba8caf997307fc53591f05661
+  metadata.gz: 89680abe072d9ece72ed12e15a8301aca55c7a8a3ec5f6ee9fb092ebd507eeac
+  data.tar.gz: d65dddefd46aa15b4408cd1ba952faf24d603edfb59b5b1b556e46ea982875d7
 SHA512:
-  metadata.gz: 785e3c7485886be0fd7946b61463e2e2b66319cf4c4f30980b6e4b2b0b8ed2ae4348be41a48caa9fbad536901a89e9c73d4d4fd005283bc2cc0dc9c9102d6ff6
-  data.tar.gz: 7a331ed220702dbab23bd6d16637f33a3da2f431a17939647ee7d1fd21898b9cf76a4594b26300eb01f8c9ad1b50d0cabcd298cdbff9c363a01a7f7bba6229a7
+  metadata.gz: 882dd1f0cd4f57a90d55c1c06ba67e3f5595e68fe8908eb40069d1dbf9215e72685ddf48c832ebe48b510aac1063cfe2ed251c7dc5dd06130b56fd357c432237
+  data.tar.gz: fcb62c4fe88773b8224938a35fc254a7c0eb22845e3084f257b3fa1ed8c49b95da7aa6c81d7531868cfc0d772db594bd2098884dce8f4a4bc98ea37358f359d7

data/.version

@@ -1 +1 @@
-0.2.0
+0.4.0

data/README.md

@@ -29,6 +29,7 @@ doc = NEU::MODS::Document.parse(xml_string)
 # Projection (plain data)
 doc.plain_title    # => "What's New - How We Respond to Disaster, Episode 1"
 doc.title_parts    # => { non_sort:, subtitle:, title:, part_name:, part_number: }
+                   #    byte-faithful -- the edit forms pre-fill from these
 doc.abstract       # => normalized, paragraph-joined String
 doc.topical_subjects # => ["Civil society", ...]   (every <topic>, for the access copy)
 doc.keywords       # => [...]   (only the editable attribute-free keyword subjects)
@@ -70,6 +71,19 @@ keyword-subject curated-vs-editable split. `build_*_name`'s `role:` defaults to
   the JSON/Solr access copy (dash/smart-punctuation transliteration, control
   stripping, paragraph handling). The XML preservation copy is never touched.
 
+Titles and prose share the one freetext vocabulary: `to_h[:main_title]` is
+normalized like `abstract`, so an invisible format mark, a Windows-1252 control
+or an exotic space cannot reach Solr or a display template.
+
+**The boundary matters.** Normalization belongs on projections that only feed
+display and the index. `title_parts` is deliberately *not* normalized, because
+Cerberus pre-fills its Metadata and Advanced forms from it and `MODSMerge` writes
+the posted value back into the MODS XML — cleaning there would rewrite the
+curator's own characters in the preservation copy on the next save. Cerberus
+makes the same call for prose: its editable source is the bare `<abstract>` node,
+not `doc.abstract`. Add a normalized *sibling* rather than normalizing a
+projection an edit form reads.
+
 ## Behavior fidelity & known caveats
 
 The projection is **behavior-preserving** with Atlas's prior `mods`-gem-based

data/lib/neu/mods/canonicalize.rb

@@ -37,9 +37,11 @@ module NEU
     #
     # Pipeline: force UTF-8 + scrub invalid bytes; NFC; map Unicode dashes to '-'
     # (swung-dash to '~'); transliterate the General Punctuation block (smart
-    # quotes, ellipsis, etc.) to ASCII; strip C0/C1 controls (keeping tab/newline);
-    # collapse horizontal-whitespace runs to one space; for paragraph fields,
-    # collapse 2+ newlines to exactly two; strip.
+    # quotes, ellipsis, etc.) to ASCII; map the separator controls to a newline
+    # and drop what is invisible (the soft hyphen, the rest of C0/C1, keeping
+    # tab/newline); collapse
+    # horizontal-whitespace runs to one space; for paragraph fields, collapse
+    # 2+ newlines to exactly two; strip.
     #
     #   .normalize(str)            -- single-line fields (newlines -> spaces)
     #   .normalize_paragraphs(str) -- fields that may carry paragraph breaks
@@ -56,7 +58,7 @@ module NEU
       # NOTE: U+2053 (swung dash) is intentionally excluded from dashes -- it is
       # named "dash" but conventionally maps to ASCII '~', not '-' (V1 prior art).
       DASH_CODEPOINTS = [
-        0x002D, 0x00AD, 0x058A, 0x05BE, 0x1400, 0x1806,
+        0x002D, 0x058A, 0x05BE, 0x1400, 0x1806,
         0x2010, 0x2011, 0x2012, 0x2013, 0x2014, 0x2015,
         0x2043, 0x207B, 0x208B, 0x2212,
         0x2E17, 0x2E1A, 0x2E3A, 0x2E3B, 0x2E40,
@@ -67,9 +69,28 @@ module NEU
 
       SWUNG_DASH_RE = Regexp.new(format('\\u%04X', 0x2053)).freeze
 
-      # C0 (U+0000..U+0008, U+000B..U+001F) and C1 (U+007F..U+009F). U+0009 (tab)
-      # and U+000A (newline) are preserved.
-      CONTROL_CODEPOINTS = ((0x0000..0x0008).to_a + (0x000B..0x001F).to_a + (0x007F..0x009F).to_a).freeze
+      # U+00AD is a hint about where a word may break, not a dash: it renders as
+      # nothing, and Solr discards it, so "co<00AD>operation" already matches a
+      # search for "cooperation". Mapping it to an ASCII hyphen instead would
+      # index "co" and "operation" as two tokens and lose the word, so it is
+      # dropped and kept out of DASH_CODEPOINTS.
+      SOFT_HYPHEN_RE = Regexp.new(format('\\u%04X', 0x00AD)).freeze
+
+      # U+000B (vertical tab) and U+000C (form feed) separate words rather than
+      # meaning nothing: Word writes a manual line break as U+000B and a page
+      # break as U+000C. They map to a newline, because deleting one runs the
+      # words either side of it together -- normalize_paragraphs then reads that
+      # newline as the soft wrap the line break was, and normalize turns it into
+      # a space.
+      SEPARATOR_CONTROL_CODEPOINTS = [0x000B, 0x000C].freeze
+      SEPARATOR_CONTROL_RE = char_class(SEPARATOR_CONTROL_CODEPOINTS).freeze
+
+      # C0 (U+0000..U+0008, U+000D..U+001F) and C1 (U+007F..U+009F) -- what is
+      # left once the separators above are accounted for, and none of it carries
+      # meaning in curator text. U+0009 (tab) and U+000A (newline) are preserved.
+      # U+000D is not: dropping it reduces a CRLF line ending to the single
+      # newline it stands for.
+      CONTROL_CODEPOINTS = ((0x0000..0x0008).to_a + (0x000D..0x001F).to_a + (0x007F..0x009F).to_a).freeze
       CONTROL_RE = char_class(CONTROL_CODEPOINTS).freeze
 
       HORIZONTAL_WS_CODEPOINTS = [
@@ -136,6 +157,8 @@ module NEU
         s = s.gsub(DASH_RE, "-")
         s = s.gsub(SWUNG_DASH_RE, "~")
         s = s.gsub(GENERAL_PUNCTUATION_RE) { |c| GENERAL_PUNCTUATION.fetch(c, c) }
+        s = s.gsub(SEPARATOR_CONTROL_RE, "\n")
+        s = s.gsub(SOFT_HYPHEN_RE, "")
         s.gsub(CONTROL_RE, "")
       end
     end

data/lib/neu/mods/projection.rb

@@ -15,8 +15,15 @@ module NEU
     module Projection
       # --- Title ---------------------------------------------------------------
 
-      # Structured primary-title parts. nil for an absent part (the Cerberus form
-      # treats nil as "not present"); to_h coerces to "" for the Atlas main_title.
+      # Structured primary-title parts, byte-faithful to the document. nil for an
+      # absent part (the Cerberus form treats nil as "not present"); to_h coerces
+      # to "" for the Atlas main_title.
+      #
+      # Faithful on purpose: this is what Cerberus pre-fills its edit forms from
+      # (MODSFields for the Metadata tab, load_advanced! for the Advanced tab),
+      # and MODSMerge writes back whatever the form posts. Normalising here would
+      # rewrite the curator's characters in the preservation XML on the next save.
+      # #access_title_parts is the normalised surface.
       def title_parts
         ti = primary_title_info
         {
@@ -45,7 +52,32 @@ module NEU
 
         optional = { ": " => parts[:subtitle], " - " => parts[:part_name], ", " => parts[:part_number] }
         suffix = optional.filter_map { |sep, val| "#{sep}#{val}" unless val.to_s.strip.empty? }.join
-        "#{parts[:non_sort]}#{parts[:title]}#{suffix}"
+        "#{join_non_sort(parts[:non_sort], parts[:title])}#{suffix}"
+      end
+
+      # Characters that bind a nonSort to the word after it. An elided article
+      # takes no space -- "L'Etranger", not "L' Etranger" -- and the same holds
+      # for a hyphenated prefix. U+2019 is the curly apostrophe, escaped rather
+      # than literal to keep lib/ pure ASCII (see the source-purity spec).
+      NON_SORT_BINDING = ["'", "\u2019", "-"].freeze
+
+      # MODS says a nonSort carries whatever separator it needs, so the historical
+      # composition simply concatenated. That only holds while the authored
+      # trailing space survives, and it does not: #child_text canonicalizes
+      # whitespace on read, so `<nonSort>The </nonSort>` arrives here as "The" and
+      # the title came out as "TheHobbit". Composing the separator instead makes
+      # the output right whether or not the source kept one -- which matters,
+      # because an invisible trailing space is not something a curator, a
+      # hand-edit or a third-party producer can be relied on to preserve.
+      #
+      # Callers that DO pass the space (Atlas's access-copy model) are unaffected:
+      # a nonSort already ending in whitespace is joined as-is.
+      def self.join_non_sort(non_sort, title)
+        prefix = non_sort.to_s
+        return title.to_s if prefix.empty?
+        return "#{prefix}#{title}" if prefix.end_with?(" ") || prefix.end_with?(*NON_SORT_BINDING)
+
+        "#{prefix} #{title}"
       end
 
       # --- Abstract / access ---------------------------------------------------
@@ -161,7 +193,7 @@ module NEU
       # names -- a drop-in source for `convert_xml_to_json`.
       def to_h
         {
-          main_title: title_parts.transform_values(&:to_s),
+          main_title: access_title_parts,
           names: names,
           languages: languages,
           date_created: date_created,
@@ -179,6 +211,14 @@ module NEU
         }
       end
 
+      # The title parts as the access copy wants them: normalised like the
+      # abstract, so a curly quote, an invisible format mark or a Windows-1252
+      # control cannot reach Solr or a display template. Titles and prose share
+      # one vocabulary -- the asymmetry where only prose was cleaned was the bug.
+      def access_title_parts
+        title_parts.transform_values { |value| NEU::MODS.normalize(value.to_s) }
+      end
+
       private
 
       # --- helpers -------------------------------------------------------------

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: neu-mods
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.4.0
 platform: ruby
 authors:
 - David Cliff
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-06-09 00:00:00.000000000 Z
+date: 2026-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -92,7 +92,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: Northeastern-flavored MODS XML projection + selection for the DRS.