neu-mods 0.3.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ebf2bf82cd0b9c0e5593307ecc9a986cefcb16f4035a5004f233bdd7f9d4d8ea
-  data.tar.gz: 7087e35df1b2f34430a53acfa49e9c97be12dd9b8bdc36b3ffa69d05e98189e0
+  metadata.gz: 1469790b96ff15516d9766db6673159fb9ad14128e02c4399842580c198a1740
+  data.tar.gz: 97c442fbd0765c6de753bbb140da3fa2ac61fedcb63104d38801c569a876ddcd
 SHA512:
-  metadata.gz: 63b83e54cdf64af06856fe1de2cb3763cc36a1f1f0a73403fb7836fac9e910dcd2f0ebd0a4fd5fb928b7714ddec20ad281ac1cc42b47a18fcda21cbc8ad8589f
-  data.tar.gz: 3d6ecb5f48c1c4cff669add4adc7bfa2a90bf29045b17bb1b1047a58f02820c85fbe303ec0c0e7665f7045e27d03531fe32759f6bc023d91032038baf4466c65
+  metadata.gz: 5b2fdf0e42f5fd65fccbc732ddf4a15b3f311f4719a3a95a294517da643c8005dcbb3c25a7b3e2b1e7afcb67c1f29f9cecacf7ec157640ef3a2b2d56e13197ac
+  data.tar.gz: 4ffc16684b6176a667fcf62dadc6e20c5fed2733c1f6bdcd39df345fe6df77f5cd412f62fe879676f1075f7c690ca6cb368b6876bfca0e4a98477af8112731c2

data/.version

@@ -1 +1 @@
-0.3.0
+0.5.0

data/README.md

@@ -29,9 +29,15 @@ doc = NEU::MODS::Document.parse(xml_string)
 # Projection (plain data)
 doc.plain_title    # => "What's New - How We Respond to Disaster, Episode 1"
 doc.title_parts    # => { non_sort:, subtitle:, title:, part_name:, part_number: }
+                   #    byte-faithful -- the edit forms pre-fill from these
 doc.abstract       # => normalized, paragraph-joined String
 doc.topical_subjects # => ["Civil society", ...]   (every <topic>, for the access copy)
 doc.keywords       # => [...]   (only the editable attribute-free keyword subjects)
+doc.date_created_with_precision
+                   # => [DateTime, "year"|"month"|"day"]   w3cdtf YYYY, YYYY-MM
+                   #    and YYYY-MM-DD all parse; the precision says which shape
+                   #    the record declared, so display cannot invent a month or
+                   #    a day the record never claimed
 doc.to_h           # => full projection, keyed to Atlas's Metadata::MODS attributes
 
 # Pure title composition (no document needed) — for callers that already hold
@@ -70,6 +76,19 @@ keyword-subject curated-vs-editable split. `build_*_name`'s `role:` defaults to
   the JSON/Solr access copy (dash/smart-punctuation transliteration, control
   stripping, paragraph handling). The XML preservation copy is never touched.
 
+Titles and prose share the one freetext vocabulary: `to_h[:main_title]` is
+normalized like `abstract`, so an invisible format mark, a Windows-1252 control
+or an exotic space cannot reach Solr or a display template.
+
+**The boundary matters.** Normalization belongs on projections that only feed
+display and the index. `title_parts` is deliberately *not* normalized, because
+Cerberus pre-fills its Metadata and Advanced forms from it and `MODSMerge` writes
+the posted value back into the MODS XML — cleaning there would rewrite the
+curator's own characters in the preservation copy on the next save. Cerberus
+makes the same call for prose: its editable source is the bare `<abstract>` node,
+not `doc.abstract`. Add a normalized *sibling* rather than normalizing a
+projection an edit form reads.
+
 ## Behavior fidelity & known caveats
 
 The projection is **behavior-preserving** with Atlas's prior `mods`-gem-based

data/lib/neu/mods/canonicalize.rb

@@ -37,9 +37,11 @@ module NEU
     #
     # Pipeline: force UTF-8 + scrub invalid bytes; NFC; map Unicode dashes to '-'
     # (swung-dash to '~'); transliterate the General Punctuation block (smart
-    # quotes, ellipsis, etc.) to ASCII; strip C0/C1 controls (keeping tab/newline);
-    # collapse horizontal-whitespace runs to one space; for paragraph fields,
-    # collapse 2+ newlines to exactly two; strip.
+    # quotes, ellipsis, etc.) to ASCII; map the separator controls to a newline
+    # and drop what is invisible (the soft hyphen, the rest of C0/C1, keeping
+    # tab/newline); collapse
+    # horizontal-whitespace runs to one space; for paragraph fields, collapse
+    # 2+ newlines to exactly two; strip.
     #
     #   .normalize(str)            -- single-line fields (newlines -> spaces)
     #   .normalize_paragraphs(str) -- fields that may carry paragraph breaks
@@ -56,7 +58,7 @@ module NEU
       # NOTE: U+2053 (swung dash) is intentionally excluded from dashes -- it is
       # named "dash" but conventionally maps to ASCII '~', not '-' (V1 prior art).
       DASH_CODEPOINTS = [
-        0x002D, 0x00AD, 0x058A, 0x05BE, 0x1400, 0x1806,
+        0x002D, 0x058A, 0x05BE, 0x1400, 0x1806,
         0x2010, 0x2011, 0x2012, 0x2013, 0x2014, 0x2015,
         0x2043, 0x207B, 0x208B, 0x2212,
         0x2E17, 0x2E1A, 0x2E3A, 0x2E3B, 0x2E40,
@@ -67,9 +69,28 @@ module NEU
 
       SWUNG_DASH_RE = Regexp.new(format('\\u%04X', 0x2053)).freeze
 
-      # C0 (U+0000..U+0008, U+000B..U+001F) and C1 (U+007F..U+009F). U+0009 (tab)
-      # and U+000A (newline) are preserved.
-      CONTROL_CODEPOINTS = ((0x0000..0x0008).to_a + (0x000B..0x001F).to_a + (0x007F..0x009F).to_a).freeze
+      # U+00AD is a hint about where a word may break, not a dash: it renders as
+      # nothing, and Solr discards it, so "co<00AD>operation" already matches a
+      # search for "cooperation". Mapping it to an ASCII hyphen instead would
+      # index "co" and "operation" as two tokens and lose the word, so it is
+      # dropped and kept out of DASH_CODEPOINTS.
+      SOFT_HYPHEN_RE = Regexp.new(format('\\u%04X', 0x00AD)).freeze
+
+      # U+000B (vertical tab) and U+000C (form feed) separate words rather than
+      # meaning nothing: Word writes a manual line break as U+000B and a page
+      # break as U+000C. They map to a newline, because deleting one runs the
+      # words either side of it together -- normalize_paragraphs then reads that
+      # newline as the soft wrap the line break was, and normalize turns it into
+      # a space.
+      SEPARATOR_CONTROL_CODEPOINTS = [0x000B, 0x000C].freeze
+      SEPARATOR_CONTROL_RE = char_class(SEPARATOR_CONTROL_CODEPOINTS).freeze
+
+      # C0 (U+0000..U+0008, U+000D..U+001F) and C1 (U+007F..U+009F) -- what is
+      # left once the separators above are accounted for, and none of it carries
+      # meaning in curator text. U+0009 (tab) and U+000A (newline) are preserved.
+      # U+000D is not: dropping it reduces a CRLF line ending to the single
+      # newline it stands for.
+      CONTROL_CODEPOINTS = ((0x0000..0x0008).to_a + (0x000D..0x001F).to_a + (0x007F..0x009F).to_a).freeze
       CONTROL_RE = char_class(CONTROL_CODEPOINTS).freeze
 
       HORIZONTAL_WS_CODEPOINTS = [
@@ -136,6 +157,8 @@ module NEU
         s = s.gsub(DASH_RE, "-")
         s = s.gsub(SWUNG_DASH_RE, "~")
         s = s.gsub(GENERAL_PUNCTUATION_RE) { |c| GENERAL_PUNCTUATION.fetch(c, c) }
+        s = s.gsub(SEPARATOR_CONTROL_RE, "\n")
+        s = s.gsub(SOFT_HYPHEN_RE, "")
         s.gsub(CONTROL_RE, "")
       end
     end

data/lib/neu/mods/projection.rb

@@ -15,8 +15,15 @@ module NEU
     module Projection
       # --- Title ---------------------------------------------------------------
 
-      # Structured primary-title parts. nil for an absent part (the Cerberus form
-      # treats nil as "not present"); to_h coerces to "" for the Atlas main_title.
+      # Structured primary-title parts, byte-faithful to the document. nil for an
+      # absent part (the Cerberus form treats nil as "not present"); to_h coerces
+      # to "" for the Atlas main_title.
+      #
+      # Faithful on purpose: this is what Cerberus pre-fills its edit forms from
+      # (MODSFields for the Metadata tab, load_advanced! for the Advanced tab),
+      # and MODSMerge writes back whatever the form posts. Normalising here would
+      # rewrite the curator's characters in the preservation XML on the next save.
+      # #access_title_parts is the normalised surface.
       def title_parts
         ti = primary_title_info
         {
@@ -164,32 +171,48 @@ module NEU
         node && clean(node.text)
       end
 
-      # Parsed dateCreated, or nil if no originInfo/dateCreated, or "" if present
-      # but unparseable (mirrors Atlas's safe_date_parse rescue).
-      def date_created
+      # The three w3cdtf date shapes a dateCreated may stop at: year, year-month,
+      # or a full date. Matching the shape explicitly, rather than widening
+      # DateTime.parse, is what lets the declared precision fall out of the parse
+      # instead of being guessed after it.
+      W3CDTF_DATE = /\A(\d{4})(?:-(\d{2})(?:-(\d{2}))?)?\z/
+
+      # Parsed dateCreated paired with the granularity the record declared, as
+      # [value, precision]. value is nil if no originInfo/dateCreated, or "" if
+      # present but unparseable (mirrors Atlas's safe_date_parse rescue).
+      # precision is "year", "month" or "day", and nil whenever value is not a
+      # DateTime.
+      #
+      # The precision has to be captured here, at the only point where the shape
+      # is still visible: a year-only date parses to January 1st, and no consumer
+      # downstream can tell that month and day from a record that claimed them.
+      # A preservation repository must not project a precision it was not given.
+      def date_created_with_precision
         node = doc.at_xpath("/mods:mods/mods:originInfo/mods:dateCreated", NAMESPACE)
-        return nil unless node
+        return [nil, nil] unless node
 
         str = NEU::MODS.canonical_ws(node.text)
-        return nil if str.empty?
+        return [nil, nil] if str.empty?
 
-        begin
-          DateTime.parse(str)
-        rescue Date::Error
-          ""
-        end
+        parse_w3cdtf(str)
       end
 
+      def date_created = date_created_with_precision.first
+      def date_created_precision = date_created_with_precision.last
+
       # --- Full projection -----------------------------------------------------
 
       # The complete read projection, keyed to Atlas's Metadata::MODS attribute
       # names -- a drop-in source for `convert_xml_to_json`.
       def to_h
+        date, date_precision = date_created_with_precision
+
         {
-          main_title: title_parts.transform_values(&:to_s),
+          main_title: access_title_parts,
           names: names,
           languages: languages,
-          date_created: date_created,
+          date_created: date,
+          date_created_precision: date_precision,
           resource_type: resource_type,
           genres: genres,
           format: format,
@@ -204,10 +227,38 @@ module NEU
         }
       end
 
+      # The title parts as the access copy wants them: normalised like the
+      # abstract, so a curly quote, an invisible format mark or a Windows-1252
+      # control cannot reach Solr or a display template. Titles and prose share
+      # one vocabulary -- the asymmetry where only prose was cleaned was the bug.
+      def access_title_parts
+        title_parts.transform_values { |value| NEU::MODS.normalize(value.to_s) }
+      end
+
       private
 
       # --- helpers -------------------------------------------------------------
 
+      # A shape-matched but impossible date (2026-13, 2026-02-30) reaches DateTime
+      # and raises; it falls to the "" sentinel like any other unparseable value.
+      # Anything outside the three shapes keeps the old permissive parse, so a
+      # timestamp still projects as a full date.
+      def parse_w3cdtf(str)
+        m = W3CDTF_DATE.match(str)
+        return [DateTime.parse(str), "day"] unless m
+
+        precision = if m[3]
+                      "day"
+                    elsif m[2]
+                      "month"
+                    else
+                      "year"
+                    end
+        [DateTime.new(m[1].to_i, (m[2] || 1).to_i, (m[3] || 1).to_i), precision]
+      rescue Date::Error
+        ["", nil]
+      end
+
       def text_at(xpath)
         node = doc.at_xpath(xpath, NAMESPACE)
         node ? NEU::MODS.canonical_ws(node.text) : ""

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: neu-mods
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.5.0
 platform: ruby
 authors:
 - David Cliff
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-08-03 00:00:00.000000000 Z
+date: 2026-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -92,7 +92,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: Northeastern-flavored MODS XML projection + selection for the DRS.