neu-mods 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ebf2bf82cd0b9c0e5593307ecc9a986cefcb16f4035a5004f233bdd7f9d4d8ea
-  data.tar.gz: 7087e35df1b2f34430a53acfa49e9c97be12dd9b8bdc36b3ffa69d05e98189e0
+  metadata.gz: 89680abe072d9ece72ed12e15a8301aca55c7a8a3ec5f6ee9fb092ebd507eeac
+  data.tar.gz: d65dddefd46aa15b4408cd1ba952faf24d603edfb59b5b1b556e46ea982875d7
 SHA512:
-  metadata.gz: 63b83e54cdf64af06856fe1de2cb3763cc36a1f1f0a73403fb7836fac9e910dcd2f0ebd0a4fd5fb928b7714ddec20ad281ac1cc42b47a18fcda21cbc8ad8589f
-  data.tar.gz: 3d6ecb5f48c1c4cff669add4adc7bfa2a90bf29045b17bb1b1047a58f02820c85fbe303ec0c0e7665f7045e27d03531fe32759f6bc023d91032038baf4466c65
+  metadata.gz: 882dd1f0cd4f57a90d55c1c06ba67e3f5595e68fe8908eb40069d1dbf9215e72685ddf48c832ebe48b510aac1063cfe2ed251c7dc5dd06130b56fd357c432237
+  data.tar.gz: fcb62c4fe88773b8224938a35fc254a7c0eb22845e3084f257b3fa1ed8c49b95da7aa6c81d7531868cfc0d772db594bd2098884dce8f4a4bc98ea37358f359d7

data/.version

@@ -1 +1 @@
-0.3.0
+0.4.0

data/README.md

@@ -29,6 +29,7 @@ doc = NEU::MODS::Document.parse(xml_string)
 # Projection (plain data)
 doc.plain_title    # => "What's New - How We Respond to Disaster, Episode 1"
 doc.title_parts    # => { non_sort:, subtitle:, title:, part_name:, part_number: }
+                   #    byte-faithful -- the edit forms pre-fill from these
 doc.abstract       # => normalized, paragraph-joined String
 doc.topical_subjects # => ["Civil society", ...]   (every <topic>, for the access copy)
 doc.keywords       # => [...]   (only the editable attribute-free keyword subjects)
@@ -70,6 +71,19 @@ keyword-subject curated-vs-editable split. `build_*_name`'s `role:` defaults to
   the JSON/Solr access copy (dash/smart-punctuation transliteration, control
   stripping, paragraph handling). The XML preservation copy is never touched.
 
+Titles and prose share the one freetext vocabulary: `to_h[:main_title]` is
+normalized like `abstract`, so an invisible format mark, a Windows-1252 control
+or an exotic space cannot reach Solr or a display template.
+
+**The boundary matters.** Normalization belongs on projections that only feed
+display and the index. `title_parts` is deliberately *not* normalized, because
+Cerberus pre-fills its Metadata and Advanced forms from it and `MODSMerge` writes
+the posted value back into the MODS XML — cleaning there would rewrite the
+curator's own characters in the preservation copy on the next save. Cerberus
+makes the same call for prose: its editable source is the bare `<abstract>` node,
+not `doc.abstract`. Add a normalized *sibling* rather than normalizing a
+projection an edit form reads.
+
 ## Behavior fidelity & known caveats
 
 The projection is **behavior-preserving** with Atlas's prior `mods`-gem-based

data/lib/neu/mods/canonicalize.rb

@@ -37,9 +37,11 @@ module NEU
     #
     # Pipeline: force UTF-8 + scrub invalid bytes; NFC; map Unicode dashes to '-'
     # (swung-dash to '~'); transliterate the General Punctuation block (smart
-    # quotes, ellipsis, etc.) to ASCII; strip C0/C1 controls (keeping tab/newline);
-    # collapse horizontal-whitespace runs to one space; for paragraph fields,
-    # collapse 2+ newlines to exactly two; strip.
+    # quotes, ellipsis, etc.) to ASCII; map the separator controls to a newline
+    # and drop what is invisible (the soft hyphen, the rest of C0/C1, keeping
+    # tab/newline); collapse
+    # horizontal-whitespace runs to one space; for paragraph fields, collapse
+    # 2+ newlines to exactly two; strip.
     #
     #   .normalize(str)            -- single-line fields (newlines -> spaces)
     #   .normalize_paragraphs(str) -- fields that may carry paragraph breaks
@@ -56,7 +58,7 @@ module NEU
       # NOTE: U+2053 (swung dash) is intentionally excluded from dashes -- it is
       # named "dash" but conventionally maps to ASCII '~', not '-' (V1 prior art).
       DASH_CODEPOINTS = [
-        0x002D, 0x00AD, 0x058A, 0x05BE, 0x1400, 0x1806,
+        0x002D, 0x058A, 0x05BE, 0x1400, 0x1806,
         0x2010, 0x2011, 0x2012, 0x2013, 0x2014, 0x2015,
         0x2043, 0x207B, 0x208B, 0x2212,
         0x2E17, 0x2E1A, 0x2E3A, 0x2E3B, 0x2E40,
@@ -67,9 +69,28 @@ module NEU
 
       SWUNG_DASH_RE = Regexp.new(format('\\u%04X', 0x2053)).freeze
 
-      # C0 (U+0000..U+0008, U+000B..U+001F) and C1 (U+007F..U+009F). U+0009 (tab)
-      # and U+000A (newline) are preserved.
-      CONTROL_CODEPOINTS = ((0x0000..0x0008).to_a + (0x000B..0x001F).to_a + (0x007F..0x009F).to_a).freeze
+      # U+00AD is a hint about where a word may break, not a dash: it renders as
+      # nothing, and Solr discards it, so "co<00AD>operation" already matches a
+      # search for "cooperation". Mapping it to an ASCII hyphen instead would
+      # index "co" and "operation" as two tokens and lose the word, so it is
+      # dropped and kept out of DASH_CODEPOINTS.
+      SOFT_HYPHEN_RE = Regexp.new(format('\\u%04X', 0x00AD)).freeze
+
+      # U+000B (vertical tab) and U+000C (form feed) separate words rather than
+      # meaning nothing: Word writes a manual line break as U+000B and a page
+      # break as U+000C. They map to a newline, because deleting one runs the
+      # words either side of it together -- normalize_paragraphs then reads that
+      # newline as the soft wrap the line break was, and normalize turns it into
+      # a space.
+      SEPARATOR_CONTROL_CODEPOINTS = [0x000B, 0x000C].freeze
+      SEPARATOR_CONTROL_RE = char_class(SEPARATOR_CONTROL_CODEPOINTS).freeze
+
+      # C0 (U+0000..U+0008, U+000D..U+001F) and C1 (U+007F..U+009F) -- what is
+      # left once the separators above are accounted for, and none of it carries
+      # meaning in curator text. U+0009 (tab) and U+000A (newline) are preserved.
+      # U+000D is not: dropping it reduces a CRLF line ending to the single
+      # newline it stands for.
+      CONTROL_CODEPOINTS = ((0x0000..0x0008).to_a + (0x000D..0x001F).to_a + (0x007F..0x009F).to_a).freeze
       CONTROL_RE = char_class(CONTROL_CODEPOINTS).freeze
 
       HORIZONTAL_WS_CODEPOINTS = [
@@ -136,6 +157,8 @@ module NEU
         s = s.gsub(DASH_RE, "-")
         s = s.gsub(SWUNG_DASH_RE, "~")
         s = s.gsub(GENERAL_PUNCTUATION_RE) { |c| GENERAL_PUNCTUATION.fetch(c, c) }
+        s = s.gsub(SEPARATOR_CONTROL_RE, "\n")
+        s = s.gsub(SOFT_HYPHEN_RE, "")
         s.gsub(CONTROL_RE, "")
       end
     end

data/lib/neu/mods/projection.rb

@@ -15,8 +15,15 @@ module NEU
     module Projection
       # --- Title ---------------------------------------------------------------
 
-      # Structured primary-title parts. nil for an absent part (the Cerberus form
-      # treats nil as "not present"); to_h coerces to "" for the Atlas main_title.
+      # Structured primary-title parts, byte-faithful to the document. nil for an
+      # absent part (the Cerberus form treats nil as "not present"); to_h coerces
+      # to "" for the Atlas main_title.
+      #
+      # Faithful on purpose: this is what Cerberus pre-fills its edit forms from
+      # (MODSFields for the Metadata tab, load_advanced! for the Advanced tab),
+      # and MODSMerge writes back whatever the form posts. Normalising here would
+      # rewrite the curator's characters in the preservation XML on the next save.
+      # #access_title_parts is the normalised surface.
       def title_parts
         ti = primary_title_info
         {
@@ -186,7 +193,7 @@ module NEU
       # names -- a drop-in source for `convert_xml_to_json`.
       def to_h
         {
-          main_title: title_parts.transform_values(&:to_s),
+          main_title: access_title_parts,
           names: names,
           languages: languages,
           date_created: date_created,
@@ -204,6 +211,14 @@ module NEU
         }
       end
 
+      # The title parts as the access copy wants them: normalised like the
+      # abstract, so a curly quote, an invisible format mark or a Windows-1252
+      # control cannot reach Solr or a display template. Titles and prose share
+      # one vocabulary -- the asymmetry where only prose was cleaned was the bug.
+      def access_title_parts
+        title_parts.transform_values { |value| NEU::MODS.normalize(value.to_s) }
+      end
+
       private
 
       # --- helpers -------------------------------------------------------------

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: neu-mods
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - David Cliff
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-08-03 00:00:00.000000000 Z
+date: 2026-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -92,7 +92,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: Northeastern-flavored MODS XML projection + selection for the DRS.