traject 1.0.0.beta.1 → 1.0.0.beta.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 19fae7c4d428bc40fd5c568f13daa7583fcebb2a
-  data.tar.gz: 9c71c57d86f6b9585451e958596a757cc75c432d
+  metadata.gz: cb35b4c5ba302cb865b459bfac6859ef6be68927
+  data.tar.gz: 14964b88428d0a827932cbf17194a77c56de1091
 SHA512:
-  metadata.gz: 6730cd527d1ea285a5d60ebddeb95950dae2fdb5c548b905627eddf7a4a985e1891a039f409577903f181c60613861a3798fede6d5ecc8a942140cbb7a2d63d4
-  data.tar.gz: ee66b43f7bed57fcb21d82e7cd82c6e49c973bf7780b7a9f9c95fb5afc28744ffdb03e63d9d9b28bf567123571fa760ea9fed30fa62d20cf0ebfd0ef95d5880a
+  metadata.gz: b3ae114fe4a11baaf6f6470d35d5df339a32b8f7e747f012310ee90ed55b982754d196abcc25ec0375d8bf988ca5e3ebd2c71ef7df159bc7007e6cdae9c69643
+  data.tar.gz: 74c6c6170860cf1cd4883d644f9c23151a703ff996592abb8668a655dcff885c108c7e04e13419f6c4ba727384b3bab3dff896c177382284b8d12b98cc711225

data/README.md

@@ -3,7 +3,7 @@
 Tools for reading MARC records, transforming them with indexing rules, and indexing to Solr.
 Might be used to index MARC data for a Solr-based discovery product like [Blacklight](https://github.com/projectblacklight/blacklight) or [VUFind](http://vufind.org/).
 
-Traject might also be generalized to a set of tools for getting structured data from a source, and sending it to a destination. 
+Traject might also be generalized to a set of tools for getting structured data from a source, and transforming it to a hash-like object to send to a destination. 
 
 
 **Traject is nearing 1.0, it is robust, feature-rich and being used in production by authors -- feedback invited**

data/lib/traject/indexer.rb

@@ -7,7 +7,6 @@ require 'traject/indexer/settings'
 require 'traject/marc_reader'
 require 'traject/marc4j_reader'
 require 'traject/json_writer'
-require 'traject/solrj_writer'
 
 require 'traject/macros/marc21'
 require 'traject/macros/basic'
@@ -71,9 +70,10 @@ require 'traject/macros/basic'
 #  4) Optionally implements a #skipped_record_count method, returning int count of records
 #     that were skipped due to errors (and presumably logged)
 #
-#  The default writer (will be) the SolrWriter , which is configured
-#  through additional Settings as well. A JsonWriter is also available,
-#  which can be useful for debugging your index mappings.
+#  The default writer is the SolrJWriter, using Java SolrJ to
+#  write to a Solr.  A few other built-in writers are available,
+#  but it's anticipated more will be created as plugins or local
+#  code for special purposes. 
 #
 #  You can set alternate writers by setting a Class object directly
 #  with the #writer_class method, or by the 'writer_class_name' Setting,

data/lib/traject/macros/marc21.rb

@@ -191,8 +191,18 @@ module Traject::Macros
     #
     # Returns altered string, doesn't change original arg.
     def self.trim_punctuation(str)
+      
+      # If something went wrong and we got a nil, just return it
+      return str unless str
+      
+      # trailing: comma, slash, semicolon, colon (possibly preceded and followed by whitespace)
       str = str.sub(/ *[ ,\/;:] *\Z/, '')
-      str = str.sub(/ *(\w\w\w)\. *\Z/, '\1')
+
+      # trailing period if it is preceded by at least three letters (possibly preceded and followed by whitespace)
+      str = str.sub(/( *\w\w\w)\. *\Z/, '\1')
+
+      # single square bracket characters if they are the start and/or end
+      #   chars and there are no internal square brackets.
       str = str.sub(/\A\[?([^\[\]]+)\]?\Z/, '\1')
       return str
     end

data/lib/traject/macros/marc21_semantics.rb

@@ -1,3 +1,5 @@
+# Encoding: UTF-8
+
 require 'traject/marc_extractor'
 
 module Traject::Macros
@@ -81,7 +83,8 @@ module Traject::Macros
     # 245 a and b, with non-filing characters stripped off
     def marc_sortable_title
       lambda do |record, accumulator|
-        accumulator << Marc21Semantics.get_sortable_title(record)
+        st = Marc21Semantics.get_sortable_title(record)
+        accumulator << st if st
       end
     end
 
@@ -503,6 +506,71 @@ module Traject::Macros
       end
     end
 
+    # Extracts LCSH-carrying fields, and formatting them
+    # as a pre-coordinated LCSH string, for instance suitable for including
+    # in a facet. 
+    #
+    # You can supply your own list of fields as a spec, but for significant
+    # customization you probably just want to write your own method in
+    # terms of the Marc21Semantics.assemble_lcsh method. 
+    def marc_lcsh_formatted(options = {})
+      spec            = options[:spec] || "600:610:611:630:648:650:651:654:6662"
+      subd_separator  = options[:subdivison_separator] || " — "
+      other_separator = options[:other_separator] || " "
+
+      extractor       = MarcExtractor.new(spec)
+
+      return lambda do |record, accumulator|
+        accumulator.concat( extractor.collect_matching_lines(record) do |field, spec|
+          Marc21Semantics.assemble_lcsh(field, subd_separator, other_separator)
+        end)
+      end
+
+    end
+
+    # Takes a MARC::Field and formats it into a pre-coordinated LCSH string
+    # with subdivision seperators in the right place. 
+    #
+    # For 600 fields especially, need to not just join with subdivision seperator
+    # to take acount of $a$d$t -- for other fields, might be able to just
+    # join subfields, not sure. 
+    #
+    # WILL strip trailing period from generated string, contrary to some LCSH practice.
+    # Our data is inconsistent on whether it has period or not, this was
+    # the easiest way to standardize. 
+    #
+    # Default subdivision seperator is em-dash with spaces, set to '--' if you want. 
+    #
+    # Cite: "Dash (-) that precedes a subdivision in an extended 600 subject heading
+    # is not carried in the MARC record. It may be system generated as a display constant
+    # associated with the content of subfield $v, $x, $y, and $z."
+    # http://www.loc.gov/marc/bibliographic/bd600.html
+    def self.assemble_lcsh(marc_field, subd_separator = " — ", other_separator = " ")
+      str = ""
+      subd_prefix_codes = %w{v x y z}
+
+
+      marc_field.subfields.each_with_index do |sf, i|
+        # ignore non-alphabetic, like numeric control subfields
+        next unless sf.code =~ /\A[a-z]\Z/
+
+        prefix = if subd_prefix_codes.include? sf.code
+          subd_separator
+        elsif i == 0
+          ""
+        else
+          other_separator
+        end
+        str << prefix << sf.value
+      end
+
+      str.gsub!(/\.\Z/, '')
+
+      return nil if str == ""
+
+      return str
+    end
+
 
   end
 end

data/lib/traject/version.rb

@@ -1,3 +1,3 @@
 module Traject
-  VERSION = "1.0.0.beta.1"
+  VERSION = "1.0.0.beta.2"
 end

data/test/indexer/macros_marc21_semantics_test.rb

@@ -1,3 +1,5 @@
+# Encoding: UTF-8
+
 require 'test_helper'
 
 require 'traject/indexer'
@@ -231,7 +233,52 @@ describe "Traject::Macros::Marc21Semantics" do
       assert_equal ["Early modern, 1500-1700", "17th century", "Great Britain: Puritan Revolution, 1642-1660", "Great Britain: Civil War, 1642-1649", "1642-1660"],
         output["era_facet"]
     end
+  end
+
+  describe "marc_lcsh_display" do
+    it "formats typical field" do      
+      field = MARC::DataField.new('650', ' ', ' ', ['a', 'Psychoanalysis and literature'], ['z', 'England'], ['x', 'History'], ['y', '19th century.'])
+      str = Marc21Semantics.assemble_lcsh(field)
+
+      assert_equal "Psychoanalysis and literature — England — History — 19th century", str
+    end
+
+    it "ignores numeric subfields" do
+      field = MARC::DataField.new('650', ' ', ' ', ['a', 'Psychoanalysis and literature'], ['x', 'History'], ['0', '01234'], ['3', 'Some part'])
+      str = Marc21Semantics.assemble_lcsh(field)
+
+      assert_equal "Psychoanalysis and literature — History", str
+    end
+
+    it "doesn't put subdivision in wrong place" do 
+      field = MARC::DataField.new('600', ' ', ' ', ['a', 'Eliot, George,'],['d', '1819-1880.'], ['t', 'Middlemarch'])
+      str = Marc21Semantics.assemble_lcsh(field)
+
+      assert_equal "Eliot, George, 1819-1880. Middlemarch", str
+    end
+
+    it "mixes non-subdivisions with subdivisions" do
+      field = MARC::DataField.new('600', ' ', ' ', ['a', 'Eliot, George,'],['d', '1819-1880.'], ['t', 'Middlemarch'], ['x', 'Criticism.'])
+      str = Marc21Semantics.assemble_lcsh(field)
+
+      assert_equal "Eliot, George, 1819-1880. Middlemarch — Criticism", str
+    end
+
+    it "returns nil for a field with no relevant subfields" do
+      field = MARC::DataField.new('650', ' ', ' ')
+      assert_nil Marc21Semantics.assemble_lcsh(field)
+    end
+
+    describe "marc_lcsh_formatted macro" do
+      it "smoke test" do
+        @record = MARC::Reader.new(support_file_path  "george_eliot.marc").to_a.first
+        @indexer.instance_eval {to_field "lcsh", marc_lcsh_formatted}
+        output = @indexer.map_record(@record)
 
+        assert output["lcsh"].length > 0, "outputs data"
+        assert output["lcsh"].include?("Eliot, George, 1819-1880 — Characters"), "includes a string its supposed to"
+      end
+    end
   end
 
   describe "extract_marc_filing_version" do
@@ -272,7 +319,6 @@ describe "Traject::Macros::Marc21Semantics" do
       end
     end
 
-
   end
 
 

data/test/indexer/macros_marc21_test.rb

@@ -97,6 +97,9 @@ describe "Traject::Macros::Marc21" do
       assert_equal "one two three", Marc21.trim_punctuation("one two three]")
       assert_equal "one two three", Marc21.trim_punctuation("[one two three")
       assert_equal "one two three", Marc21.trim_punctuation("[one two three]")
+
+      # This one was a bug before
+      assert_equal "Feminism and art", Marc21.trim_punctuation("Feminism and art.")
     end
 
     it "uses :translation_map" do

data/test/test_support/demo_config.rb

@@ -20,7 +20,7 @@ extend Traject::Macros::MarcFormats
 # files however you like, you can call traject with as many
 # config files as you like, `traject -c one.rb -c two.rb -c etc.rb`
 settings do
-  provide "solr.url", "http://blacklight.mse.jhu.edu:8983/solr/prod"
+  provide "solr.url", "http://solr.somewhere.edu:8983/solr/corename"
 
   # Only if you need to connect to a Solr 1.x:
   provide "solrj_writer.parser_class_name", "XMLResponseParser"

data/test/test_support/george_eliot.marc

@@ -0,0 +1 @@
+01359cam a2200361 a 4500001000800000005001700008008004100025010001700066020002800083020003500111035001600146040001300162043001200175049000900187050002500196082001500221100002200236245009700258260005800355300002700413440004600440504006400486600005400550600004700604600004400651600004300695650006700738650005800805650002900863910002600892994001200918991006700930232964520030805093128.0020925s2003    nyu      b   s001 0 eng    a  2002036483  a0791458334 (alk. paper)  a0791458342 (pbk. : alk. paper)  aocm50737282  aDLCcDLC  ae-uk-en  aJHEE00aPR4692.P74bP37 200300a823/.82211 aParis, Bernard J.10aRereading George Eliot :bchanging responses to her experiments in life /cBernard J. Paris.  aAlbany :bState University of New York Press,cc2003.  axiii, 220 p. ;c23 cm. 0aSUNY series in psychoanalysis and culture  aIncludes bibliographical references (p. 213-215) and index.10aEliot, George,d1819-1880xKnowledgexPsychology.10aEliot, George,d1819-1880.tDaniel Deronda10aEliot, George,d1819-1880.tMiddlemarch10aEliot, George,d1819-1880xCharacters. 0aPsychoanalysis and literaturezEnglandxHistoryy19th century. 0aPsychological fiction, EnglishxHistory and criticism 0aPsychology in literature  a2329645bHorizon bib#  aE0bJHE  aPR4692.P74 P37 2003flcbelc1cc. 1q0i3857076lembluememsel

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: traject
 version: !ruby/object:Gem::Version
-  version: 1.0.0.beta.1
+  version: 1.0.0.beta.2
 platform: ruby
 authors:
 - Jonathan Rochkind
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2013-10-14 00:00:00.000000000 Z
+date: 2013-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: marc
@@ -216,6 +216,7 @@ files:
 - test/test_support/date_with_u.marc
 - test/test_support/demo_config.rb
 - test/test_support/emptyish_record.marc
+- test/test_support/george_eliot.marc
 - test/test_support/hebrew880s.marc
 - test/test_support/louis_armstrong.marc
 - test/test_support/manufacturing_consent.marc
@@ -313,6 +314,7 @@ test_files:
 - test/test_support/date_with_u.marc
 - test/test_support/demo_config.rb
 - test/test_support/emptyish_record.marc
+- test/test_support/george_eliot.marc
 - test/test_support/hebrew880s.marc
 - test/test_support/louis_armstrong.marc
 - test/test_support/manufacturing_consent.marc