neu-mods 0.7.1 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d7e678073c1e18b2626473e1e8fcec32052a2a33075d34cea6fe96dca40e8315
-  data.tar.gz: 5d3d44f6bf9ee5600dfdaded3a9d2119472e86c1aa6621fbcd92ea8358619738
+  metadata.gz: 5416d755319f464e252ef4fa07d0372efb11e1379d76ad6155b78f42a61c221a
+  data.tar.gz: ddfcf9eb792155c216bf9710ef6cfcc379f914791a782da30577158bb02a1da7
 SHA512:
-  metadata.gz: ce304892f19636ecc57c487810a2ef2932f253d9702dcce6974a16c8e047b56ceeffb3ae25750bf2e5937345523567dd9d598ef59c32e671dcc80d62541748c8
-  data.tar.gz: a31247bacadff8b81b4789c2dc69647b2036b98e246af02492d1ecdfb14e1a625e1deb2f7c7ad7afff5298d531a6b6bcacfa803ab3ea7629ecea623e45e06667
+  metadata.gz: d465b12175d8b15f7da7d7c355af0246294b3e7df0c276be775b6ccf41c14840f716fa7624ed86a54bff69da56625041cf7ca1f245e9ff6bb8f4035b44f70799
+  data.tar.gz: ce373c9bb16d1a7e6142a4e0556e669ddb07d0bbe77db172f7954d4d54eb298b4a688c5cf3a60bd5237611670bbfb162c67d7d9c1bd5377a6579d0c78b9a9030

data/.version

@@ -1 +1 @@
-0.7.1
+0.9.0

data/README.md

@@ -34,18 +34,29 @@ doc.abstract       # => normalized, paragraph-joined String
 doc.languages      # => ["English"]   a code-only <languageTerm>eng</> included
 doc.topical_subjects # => ["Civil society", ...]   (every <topic>, for the access copy)
 doc.keywords       # => [...]   (only the editable attribute-free keyword subjects)
-doc.date_created_with_precision
-                   # => [DateTime, "year"|"month"|"day"]   w3cdtf YYYY, YYYY-MM
-                   #    and YYYY-MM-DD all parse; the precision says which shape
-                   #    the record declared, so display cannot invent a month or
-                   #    a day the record never claimed. Same for
-                   #    date_issued_with_precision and
-                   #    copyright_date_with_precision.
+doc.date_created_parts
+                   # => { value:, precision:, end_value:, end_precision:,
+                   #      qualifier:, key_date: }   everything the record
+                   #    declared about one date. w3cdtf YYYY, YYYY-MM and
+                   #    YYYY-MM-DD all parse, and the precision says which
+                   #    shape it gave, so display cannot invent a month or a
+                   #    day. The points are read by @point, not by document
+                   #    order, and the end carries its OWN precision.
+                   #    Same for date_issued_parts and copyright_date_parts;
+                   #    each part is also a reader of its own, e.g.
+                   #    doc.date_created_qualifier.
 doc.notes          # => [{ type: "funding", value: "..." }, ...]
 doc.related_items  # => [{ type: "otherFormat", title: "..." }, ...]
                    #    every relatedItem that is not a series or a host
 doc.location       # => [{ physical_location:, shelf_location:, url: }, ...]
 doc.map_data       # => [{ scale:, projection:, coordinates: }, ...]
+doc.title_subjects # => ["The Great Gatsby"]   composed like the main title
+doc.hierarchical_geographic_subjects
+                   # => [{ country:, state:, city:, ... }, ...]   eleven levels,
+                   #    structured for the reason map_data is
+doc.record_info    # => { content_source:, origin:, description_standard:,
+                   #      creation_date:, change_date:, language_of_cataloging: }
+                   #    describes the CATALOGUING, not the resource
 doc.to_h           # => full projection, keyed to Atlas's Metadata::MODS attributes
 
 # The field registry -- the single declaration of what this gem projects.
@@ -123,14 +134,17 @@ intentional notes:
   label vocabulary belongs to the consumer — Cerberus's edit form and Atlas's
   display word the same role differently. An unrecognised language code also
   stays raw, since the record still said something.
-- **`description` is not projected.** MODS has no element of that name, and the
-  two candidates — an `abstract` variant and `physicalDescription/note` —
-  describe different things. Projecting a guess would put wrong data in the
-  field rather than leave an empty one, so it waits on a decision.
-- **A date range is not projected.** `dateCreated`, `dateIssued` and
-  `copyrightDate` each project one value with its precision. A record that
-  ranges a date uses `point="start"` / `point="end"`, which needs a shape of its
-  own rather than a second array.
+- **`description` is not projected.** MODS does define `name/description`, but
+  that annotates a *name*, not the resource, so it is not the field Atlas once
+  called `description`. The two candidates for that one — an `abstract` variant
+  and `physicalDescription/note` — describe different things. Projecting a guess
+  would put wrong data in the field rather than leave an empty one, so it waits
+  on a decision.
+- **A date carries more than a value.** Each of `dateCreated`, `dateIssued` and
+  `copyrightDate` projects a value, its precision, an end value with its own
+  precision, the `@qualifier` and the `@keyDate` flag. The gem does not *pick*
+  the key date, because "which date to sort on" and "which date to display" are
+  not necessarily the same answer, and choosing is the consumer's job.
 
 ## Source convention
 

data/lib/neu/mods/projection.rb

@@ -130,16 +130,59 @@ module NEU
       def personal_name_subjects = name_subjects("personal")
       def corporate_name_subjects = name_subjects("corporate")
 
+      def genre_subjects = texts_at("/mods:mods/mods:subject/mods:genre")
+
+      # A MARC GAC code. Projected as the record wrote it: turning it into a
+      # place name needs a lookup table, which is the same call the gem already
+      # made for MARC relators -- the label vocabulary belongs to the consumer.
+      def geographic_code_subjects = texts_at("/mods:mods/mods:subject/mods:geographicCode")
+
+      # A subject that is a work has a nonSort, a subTitle and part numbers like
+      # any other titleInfo, so it composes through the same port as the main
+      # title rather than taking titleInfo/title alone.
+      def title_subjects
+        doc.xpath("/mods:mods/mods:subject/mods:titleInfo", NAMESPACE).filter_map do |node|
+          parts = title_parts_of(node).transform_values { |value| NEU::MODS.normalize(value.to_s) }
+          clean(Projection.compose_title(parts))
+        end
+      end
+
+      # Kept structured for the reason #map_data is. Flattening country / state
+      # / city into "United States -- New York (State) -- Parksville" would make
+      # a consumer that wants the city alone unpick a sentence.
+      #
+      # This is the axis bdr_43888.mods.xml uses INSTEAD of subject/geographic,
+      # so that record projected no place at all -- a live ingest path, not a
+      # hypothetical.
+      def hierarchical_geographic_subjects
+        doc.xpath("/mods:mods/mods:subject/mods:hierarchicalGeographic", NAMESPACE).filter_map do |node|
+          entry = HIERARCHICAL_GEOGRAPHIC_LEVELS.to_h { |level| [level, child_text(node, "mods:#{camelize(level)}")] }
+          entry if entry.values.any?
+        end
+      end
+
       # --- Names ---------------------------------------------------------------
 
+      # One name as the access copy wants it. `affiliation` is how a reader
+      # tells one J. Doe from another, and it is the field an institutional
+      # repository most wants: it repeats in the schema, so it is an array.
+      #
+      # Added to the entry rather than as a parallel field, so a name and its
+      # affiliation cannot be zipped together wrongly by a consumer.
+      def name_entry(node)
+        {
+          name: name_display_value_w_date(node),
+          role: name_role(node),
+          affiliation: texts_under(node, "mods:affiliation")
+        }
+      end
+
       # All top-level names as { name:, role: }. `name` reproduces the `mods` gem's
       # display_value_w_date (including its quirks -- faithfully, so existing Solr/
       # display output is preserved). `role` prefers the type="text" roleTerm,
       # falling back to the raw code (NOT MARC-relator-translated -- see README).
       def names
-        doc.xpath("/mods:mods/mods:name", NAMESPACE).map do |node|
-          { name: name_display_value_w_date(node), role: name_role(node) }
-        end
+        doc.xpath("/mods:mods/mods:name", NAMESPACE).map { |node| name_entry(node) }
       end
 
       # Editable (depositor-managed) creators: the plain names (no authority
@@ -161,7 +204,7 @@ module NEU
       def preserved_names
         doc.xpath("/mods:mods/mods:name", NAMESPACE)
            .reject { |node| editable_creator_name?(node) }
-           .map { |node| { name: name_display_value_w_date(node), role: name_role(node) } }
+           .map { |node| name_entry(node) }
       end
 
       # --- Scalars / simple arrays --------------------------------------------
@@ -196,6 +239,23 @@ module NEU
       # back by nothing.
       def publication_information = texts_at("/mods:mods/mods:originInfo/mods:publisher")
       def edition = texts_at("/mods:mods/mods:originInfo/mods:edition")
+      def place_of_publication = texts_at("/mods:mods/mods:originInfo/mods:place/mods:placeTerm")
+      def issuance = texts_at("/mods:mods/mods:originInfo/mods:issuance")
+
+      # Serials. The @authority a record puts on a frequency is not projected:
+      # authority handling is a question the gem defers everywhere else -- for
+      # genre, subject and name -- and answering it for one field would be
+      # inconsistent.
+      def frequency = texts_at("/mods:mods/mods:originInfo/mods:frequency")
+
+      def table_of_contents = texts_at("/mods:mods/mods:tableOfContents")
+      def reformatting_quality = texts_at("/mods:mods/mods:physicalDescription/mods:reformattingQuality")
+
+      # An LCC or DDC call number. Note this is NOT the same concept as Atlas's
+      # classification_ssim, which carries a FileSet content-type vocabulary --
+      # the name collision is accidental and the consumer has to pick a free
+      # Solr field.
+      def classification = texts_at("/mods:mods/mods:classification")
 
       # Every top-level note, keeping its @type. The type carries meaning -- a
       # "statement of responsibility" is not a "funding" note -- so flattening
@@ -210,11 +270,15 @@ module NEU
       # location repeats, and one location mixes kinds: a shelf mark and a URL
       # are not interchangeable, and a consumer has to know which it holds
       # before it can decide to linkify it. So the parts stay apart.
+      #
+      # The shelf mark is mods:shelfLocator. There is no shelfLocation element
+      # in MODS, and the spec fixture carried the same misspelling, so the
+      # field was unconditionally nil and the spec asserted nothing.
       def location
         doc.xpath("/mods:mods/mods:location", NAMESPACE).filter_map do |node|
           entry = {
             physical_location: child_text(node, "mods:physicalLocation"),
-            shelf_location: child_text(node, "mods:shelfLocation"),
+            shelf_location: child_text(node, "mods:shelfLocator"),
             url: child_text(node, "mods:url")
           }
           entry if entry.values.any?
@@ -259,7 +323,18 @@ module NEU
         end
       end
 
-      def identifiers = texts_at("/mods:mods/mods:identifier")
+      # { type:, value: }, because a DOI, an accession number and a collection
+      # id are not the same kind of thing and no consumer can tell them apart
+      # from the digits alone -- a reader shown a bare 10.1234/x cannot see it
+      # is a DOI, and a display cannot decide to linkify it. The same argument
+      # #notes already makes for its @type, and #permanent_url already proves
+      # the attribute is load-bearing by special-casing @type='hdl'.
+      def identifiers
+        doc.xpath("/mods:mods/mods:identifier", NAMESPACE).filter_map do |node|
+          value = clean(node.text)
+          { type: clean(node["type"]), value: value } if value
+        end
+      end
 
       def permanent_url
         node = doc.at_xpath("/mods:mods/mods:identifier[@type='hdl']", NAMESPACE)
@@ -270,34 +345,92 @@ module NEU
       # or a full date. Matching the shape explicitly, rather than widening
       # DateTime.parse, is what lets the declared precision fall out of the parse
       # instead of being guessed after it.
+      # The eleven children the XSD allows under hierarchicalGeographic, in the
+      # order MODS lists them -- broadest first, which is also the order a
+      # consumer composing a place string wants to reverse.
+      HIERARCHICAL_GEOGRAPHIC_LEVELS = %i[
+        continent country province region state territory county city
+        city_section island area
+      ].freeze
+
+      # recordInfo children. Read as a single value: the schema repeats the
+      # element, but a record with two cataloguing provenances is not a case
+      # anyone has, and an array here buys nothing.
+      RECORD_INFO_PARTS = {
+        content_source: "mods:recordContentSource",
+        origin: "mods:recordOrigin",
+        description_standard: "mods:descriptionStandard",
+        creation_date: "mods:recordCreationDate",
+        change_date: "mods:recordChangeDate",
+        language_of_cataloging: "mods:languageOfCataloging/mods:languageTerm"
+      }.freeze
+
       W3CDTF_DATE = /\A(\d{4})(?:-(\d{2})(?:-(\d{2}))?)?\z/
 
-      # The three originInfo dates, each paired with the granularity the record
-      # declared, as [value, precision]. value is nil if the element is absent,
-      # or "" if present but unparseable (mirrors Atlas's safe_date_parse
-      # rescue). precision is "year", "month" or "day", and nil whenever value
-      # is not a DateTime.
+      # What #date_parts returns when the element is absent entirely, so an
+      # absent date is distinguishable from one present and unparseable.
+      EMPTY_DATE = { value: nil, precision: nil, end_value: nil,
+                     end_precision: nil, qualifier: nil, key_date: nil }.freeze
+
+      # Everything a record declared about one originInfo date, as
+      # { value:, precision:, end_value:, end_precision:, qualifier:, key_date: }.
       #
-      # The precision has to be captured here, at the only point where the shape
-      # is still visible: a year-only date parses to January 1st, and no consumer
-      # downstream can tell that month and day from a record that claimed them.
-      # A preservation repository must not project a precision it was not given.
+      # A date is not a scalar. Precision established that: a year-only date
+      # parses to January 1st, and no consumer downstream can tell that month
+      # and day from a record that claimed them. A range and a qualifier are the
+      # same kind of claim, and dropping them breaks the same rule -- a
+      # preservation repository must not project a value the record did not
+      # give. A ranged record was worse than that: #at_xpath took the first
+      # node, so one end of the range was PROMOTED to be the date, and the
+      # output was indistinguishable from a single certain year.
       #
-      # These stay :one where the sibling originInfo elements are :many. MODS
-      # repeats them, but a date carries a precision alongside it, and an array
-      # of values paired with an array of precisions is a shape no consumer
-      # wants. A record that ranges its dates uses point="start"/"end", which is
-      # its own projection question rather than a cardinality one.
-      def date_created_with_precision = date_with_precision("dateCreated")
-      def date_issued_with_precision = date_with_precision("dateIssued")
-      def copyright_date_with_precision = date_with_precision("copyrightDate")
-
-      def date_created = date_created_with_precision.first
-      def date_created_precision = date_created_with_precision.last
-      def date_issued = date_issued_with_precision.first
-      def date_issued_precision = date_issued_with_precision.last
-      def copyright_date = copyright_date_with_precision.first
-      def copyright_date_precision = copyright_date_with_precision.last
+      # The parts are projected as separate flat fields rather than one nested
+      # value, because the value half has three consumers that need a real date
+      # object -- a Solr sort key, a citation year and an OAI date.
+      def date_created_parts = date_parts("dateCreated")
+      def date_issued_parts = date_parts("dateIssued")
+      def copyright_date_parts = date_parts("copyrightDate")
+
+      def date_created = date_created_parts[:value]
+      def date_created_precision = date_created_parts[:precision]
+      def date_created_end = date_created_parts[:end_value]
+      def date_created_end_precision = date_created_parts[:end_precision]
+      def date_created_qualifier = date_created_parts[:qualifier]
+      def date_created_key_date = date_created_parts[:key_date]
+
+      def date_issued = date_issued_parts[:value]
+      def date_issued_precision = date_issued_parts[:precision]
+      def date_issued_end = date_issued_parts[:end_value]
+      def date_issued_end_precision = date_issued_parts[:end_precision]
+      def date_issued_qualifier = date_issued_parts[:qualifier]
+      def date_issued_key_date = date_issued_parts[:key_date]
+
+      def copyright_date = copyright_date_parts[:value]
+      def copyright_date_precision = copyright_date_parts[:precision]
+      def copyright_date_end = copyright_date_parts[:end_value]
+      def copyright_date_end_precision = copyright_date_parts[:end_precision]
+      def copyright_date_qualifier = copyright_date_parts[:qualifier]
+      def copyright_date_key_date = copyright_date_parts[:key_date]
+
+      # The [value, precision] pair the precision work introduced. Retained
+      # because it is the documented entry point for a caller that wants both
+      # halves and nothing else.
+      def date_created_with_precision = [date_created, date_created_precision]
+      def date_issued_with_precision = [date_issued, date_issued_precision]
+      def copyright_date_with_precision = [copyright_date, copyright_date_precision]
+
+      # Who catalogued this record, to what standard, and when. It describes the
+      # CATALOGUING rather than the resource, which is why it is one value and
+      # why a consumer is unlikely to want it beside Publisher -- but dropping a
+      # preservation repository's provenance statement on read is wrong on its
+      # face, so it is projected and the display question is the consumer's.
+      def record_info
+        node = doc.at_xpath("/mods:mods/mods:recordInfo", NAMESPACE)
+        return nil unless node
+
+        entry = RECORD_INFO_PARTS.transform_values { |xpath| child_text(node, xpath) }
+        entry if entry.values.any?
+      end
 
       # --- Full projection -----------------------------------------------------
 
@@ -327,13 +460,30 @@ module NEU
 
         # origin
         publication_information: :many,
+        place_of_publication: :many,
         edition: :many,
+        issuance: :many,
+        frequency: :many,
+        # Six rows per originInfo date. Flat rather than one nested value,
+        # because the value half has consumers that need a real date object.
         date_created: :one,
         date_created_precision: :one,
+        date_created_end: :one,
+        date_created_end_precision: :one,
+        date_created_qualifier: :one,
+        date_created_key_date: :one,
         date_issued: :one,
         date_issued_precision: :one,
+        date_issued_end: :one,
+        date_issued_end_precision: :one,
+        date_issued_qualifier: :one,
+        date_issued_key_date: :one,
         copyright_date: :one,
         copyright_date_precision: :one,
+        copyright_date_end: :one,
+        copyright_date_end_precision: :one,
+        copyright_date_qualifier: :one,
+        copyright_date_key_date: :one,
 
         # physical description
         resource_type: :many,
@@ -341,7 +491,9 @@ module NEU
         format: :many,
         extent: :many,
         digital_origin: :many,
+        reformatting_quality: :many,
         notes: :many,
+        table_of_contents: :many,
 
         # subjects
         topical_subjects: :many,
@@ -349,6 +501,10 @@ module NEU
         temporal_subjects: :many,
         personal_name_subjects: :many,
         corporate_name_subjects: :many,
+        genre_subjects: :many,
+        geographic_code_subjects: :many,
+        title_subjects: :many,
+        hierarchical_geographic_subjects: :many,
         map_data: :many,
 
         # related items
@@ -358,7 +514,9 @@ module NEU
 
         # identifiers and location
         identifiers: :many,
+        classification: :many,
         permanent_url: :one,
+        record_info: :one,
         location: :many,
 
         # access
@@ -415,8 +573,42 @@ module NEU
         ["", nil]
       end
 
-      def date_with_precision(element)
-        node = doc.at_xpath("/mods:mods/mods:originInfo/mods:#{element}", NAMESPACE)
+      # One originInfo date element, read by its attributes rather than by
+      # position. A record is free to write point="end" first, and taking the
+      # first node would then invert the range.
+      def date_parts(element)
+        nodes = doc.xpath("/mods:mods/mods:originInfo/mods:#{element}", NAMESPACE)
+        return EMPTY_DATE if nodes.empty?
+
+        start = nodes.find { |n| attr_value(n, "point") == "start" } ||
+                nodes.find { |n| attr_value(n, "point") != "end" }
+        finish = nodes.find { |n| attr_value(n, "point") == "end" }
+        date_entry(start, finish, nodes)
+      end
+
+      # The end point carries its OWN precision. "1935-06" to "1940" is legal,
+      # and reusing the start's granularity for both would assert something the
+      # end never claimed -- the precision bug in a new place.
+      #
+      # The qualifier falls back from the start to the end because v1's loader
+      # applied it to both points, and a record marking only one is still
+      # telling us the date is uncertain. An unrecognised value survives as
+      # itself: MODS enumerates approximate, inferred and questionable, but the
+      # record still said something.
+      def date_entry(start, finish, nodes)
+        value, precision = node_date(start)
+        end_value, end_precision = node_date(finish)
+        {
+          value: value,
+          precision: precision,
+          end_value: end_value,
+          end_precision: end_precision,
+          qualifier: attr_value(start, "qualifier") || attr_value(finish, "qualifier"),
+          key_date: nodes.any? { |n| attr_value(n, "keyDate") == "yes" }
+        }
+      end
+
+      def node_date(node)
         return [nil, nil] unless node
 
         str = NEU::MODS.canonical_ws(node.text)
@@ -425,6 +617,13 @@ module NEU
         parse_w3cdtf(str)
       end
 
+      def attr_value(node, name)
+        return nil unless node
+
+        value = NEU::MODS.canonical_ws(node[name].to_s)
+        value.empty? ? nil : value
+      end
+
       # Byte-faithful title parts off any titleInfo node, shared by #title_parts
       # (which Cerberus pre-fills its edit forms from) and the variant titles.
       def title_parts_of(node)
@@ -466,6 +665,19 @@ module NEU
       # record template that seeds an empty <topic> for an edit form to fill --
       # which is exactly what Atlas's MODSBuilder writes -- otherwise projects
       # [nil], and every consumer of that array has to guard for it.
+      # #texts_at scoped to a node rather than the document, for a repeatable
+      # child of one element.
+      # :city_section -> "citySection". The level names are snake_case in the
+      # projection and camelCase in the schema.
+      def camelize(level)
+        head, *rest = level.to_s.split("_")
+        [head, *rest.map(&:capitalize)].join
+      end
+
+      def texts_under(node, xpath)
+        node.xpath(xpath, NAMESPACE).filter_map { |child| clean(child.text) }
+      end
+
       def texts_at(xpath)
         doc.xpath(xpath, NAMESPACE).filter_map { |node| clean(node.text) }
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: neu-mods
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.9.0
 platform: ruby
 authors:
 - David Cliff