traject 0.0.1

data/lib/traject/version.rb

@@ -0,0 +1,3 @@
+module Traject
+  VERSION = "0.0.1"
+end

data/test/indexer/macros_marc21_test.rb

@@ -0,0 +1,146 @@
+require 'test_helper'
+
+require 'traject/indexer'
+require 'traject/macros/marc21'
+
+require 'json'
+require 'marc/record'
+
+# See also marc_extractor_test.rb for more detailed tests on marc extraction,
+# this is just a basic test to make sure our macro works passing through to there
+# and other options.
+describe "Traject::Macros::Marc21" do
+  Marc21 = Traject::Macros::Marc21 # shortcut
+
+  before do
+    @indexer = Traject::Indexer.new
+    @indexer.extend Traject::Macros::Marc21
+    @record = MARC::Reader.new(support_file_path  "manufacturing_consent.marc").to_a.first
+  end
+
+  describe "extract_marc" do
+    it "extracts marc" do
+      @indexer.instance_eval do
+        to_field "title", extract_marc("245ab")
+      end
+
+      output = @indexer.map_record(@record)
+
+      assert_equal ["Manufacturing consent : the political economy of the mass media /"], output["title"]
+    end
+
+    it "respects :first=>true option" do
+      @indexer.instance_eval do
+        to_field "other_id", extract_marc("035a", :first => true)
+      end
+
+      output = @indexer.map_record(@record)
+
+      assert_length 1, output["other_id"]
+    end
+
+    it "trims punctuation with :trim_punctuation => true" do
+      @indexer.instance_eval do
+        to_field "title", extract_marc("245ab", :trim_punctuation => true)
+      end
+
+      output = @indexer.map_record(@record)
+
+      assert ! output["title"].first.end_with?("/"), "does not end with /"
+    end
+
+    it "Marc21::trim_punctuation class method" do
+      assert_equal "one two three", Marc21.trim_punctuation("one two three")
+
+      assert_equal "one two three", Marc21.trim_punctuation("one two three,")
+      assert_equal "one two three", Marc21.trim_punctuation("one two three/")
+      assert_equal "one two three", Marc21.trim_punctuation("one two three;")
+      assert_equal "one two three", Marc21.trim_punctuation("one two three:")
+      assert_equal "one two three .", Marc21.trim_punctuation("one two three .")
+      assert_equal "one two three", Marc21.trim_punctuation("one two three.")
+
+      assert_equal "one two [three]", Marc21.trim_punctuation("one two [three]")
+      assert_equal "one two three", Marc21.trim_punctuation("one two three]")
+      assert_equal "one two three", Marc21.trim_punctuation("[one two three")
+      assert_equal "one two three", Marc21.trim_punctuation("[one two three]")
+    end
+
+    it "uses :translation_map" do
+      @indexer.instance_eval do
+        to_field "cataloging_agency", extract_marc("040a", :seperator => nil, :translation_map => "marc_040a_translate_test")
+      end
+      output = @indexer.map_record(@record)
+
+      assert_equal ["Library of Congress"], output["cataloging_agency"]
+    end
+  end
+
+  describe "serialized_marc" do
+    it "serializes xml" do
+      @indexer.instance_eval do
+        to_field "marc_record", serialized_marc(:format => "xml")
+      end
+      output = @indexer.map_record(@record)
+
+      assert_length 1, output["marc_record"]
+      assert_kind_of String, output["marc_record"].first
+      assert output["marc_record"].first.start_with?("<record xmlns='http://www.loc.gov/MARC21/slim'>"), "looks like serialized MarcXML"
+    end
+
+    it "serializes binary UUEncoded" do
+      @indexer.instance_eval do
+        to_field "marc_record", serialized_marc(:format => "binary")
+      end
+      output = @indexer.map_record(@record)
+
+      assert_length 1, output["marc_record"]
+      assert_kind_of String, output["marc_record"].first
+
+      decoded = Base64.decode64( output["marc_record"].first )
+
+      # just check the marc header for now
+      assert_start_with "02067cam a2200469", decoded
+    end
+
+    it "serializes binary raw" do
+      @indexer.instance_eval do
+        to_field "marc_record", serialized_marc(:format => "binary", :binary_escape => false)
+      end
+      output = @indexer.map_record(@record)
+
+      assert_length 1, output["marc_record"]
+      assert_kind_of String, output["marc_record"].first
+
+      # just check the marc header for now
+      assert_start_with "02067cam a2200469", output["marc_record"].first
+    end
+
+    it "serializes json" do
+      @indexer.instance_eval do
+        to_field "marc_record", serialized_marc(:format => "json")
+      end
+      output = @indexer.map_record(@record)
+
+      assert_length 1, output["marc_record"]
+
+      # okay, let's actually deserialize it, why not
+
+      hash = JSON.parse( output["marc_record"].first )
+
+      deserialized = MARC::Record.new_from_hash(hash)
+
+      assert_equal @record, deserialized
+    end
+  end
+
+  it "#extract_all_marc_values" do
+    @indexer.instance_eval do
+      to_field "text", extract_all_marc_values
+    end
+    output = @indexer.map_record(@record)
+
+    assert_length 13, output["text"]
+  end
+
+
+end

data/test/indexer/macros_test.rb

@@ -0,0 +1,40 @@
+require 'test_helper'
+
+describe "Indexer Macros:" do
+  before do
+    @indexer = Traject::Indexer.new
+    @record = MARC::Reader.new(support_file_path  "manufacturing_consent.marc").to_a.first
+  end
+
+  it "works with simple literal" do
+    @indexer.instance_eval do
+      extend Traject::Macros::Basic
+
+      to_field "source", literal("MY LIBRARY")
+    end
+
+    output = @indexer.map_record(@record)
+
+    assert_equal ["MY LIBRARY"], output["source"]
+  end
+
+  it "works with macro AND block" do
+    called = false
+
+    @indexer.instance_eval do
+      extend Traject::Macros::Basic
+      to_field "source", literal("MY LIBRARY") do |record, accumulator, context|
+        called = true
+        accumulator << "SECOND VALUE"
+      end
+    end
+
+    output = @indexer.map_record(@record)
+
+    assert called
+    assert_equal ["MY LIBRARY", "SECOND VALUE"], output["source"]
+  end
+
+
+
+end

data/test/indexer/map_record_test.rb

@@ -0,0 +1,120 @@
+require 'test_helper'
+
+describe "Traject::Indexer#map_record" do 
+  before do
+    @indexer = Traject::Indexer.new
+    @record = MARC::Reader.new(support_file_path  "manufacturing_consent.marc").to_a.first    
+  end
+
+
+  describe "with no indexing rules" do
+    it "returns empty hash" do
+      output = @indexer.map_record(@record)
+
+      assert_kind_of Hash, output
+      assert_empty output
+    end
+  end
+
+  describe "#to_field" do    
+    it "works with block" do
+      called  = false
+
+      @indexer.to_field("title") do |record, accumulator|
+        assert_kind_of MARC::Record, record
+        assert_kind_of Array, accumulator
+
+        called = true # by the power of closure!
+        accumulator << "Some Title"
+      end
+      
+      output = @indexer.map_record(@record)
+
+      assert called
+      assert_kind_of Hash, output
+      assert_equal ["Some Title"], output["title"]
+    end
+
+    it "works with a lambda arg" do
+      called  = false
+
+      logic = lambda do |record, accumulator|
+        assert_kind_of MARC::Record, record
+        assert_kind_of Array, accumulator
+
+        called = true # by the power of closure!
+        accumulator << "Some Title"
+      end
+
+      @indexer.to_field("title", logic) 
+      
+      output = @indexer.map_record(@record)
+
+      assert called
+      assert_kind_of Hash, output
+      assert_equal ["Some Title"], output["title"]
+    end
+
+    it "works with both lambda and Proc" do
+      block_called = false
+
+      lambda_arg = lambda do |record, accumulator|
+        accumulator << "Lambda-provided Value"
+      end
+
+      @indexer.to_field("title", lambda_arg) do |record, accumulator|
+        assert_includes accumulator, "Lambda-provided Value"
+        accumulator << "Block-provided Value"
+
+        block_called = true
+      end
+
+      output = @indexer.map_record(@record)
+
+      assert block_called
+      assert_includes output["title"], "Lambda-provided Value"
+      assert_includes output["title"], "Block-provided Value"
+    end
+  end
+
+  describe "multiple to_field blocks" do
+    it "get called in order" do
+      order = []
+      @indexer.to_field("title") do |rec, acc|
+        order << :first_one
+        acc << "First"
+      end
+      @indexer.to_field("title") do |rec, acc|
+        order << :second_one
+        acc << "Second"
+      end
+
+      output = @indexer.map_record(@record)
+
+      assert_equal [:first_one, :second_one], order
+      assert_equal ["First", "Second"], output["title"]
+    end
+  end
+
+  describe "context argument" do
+    it "is third argument to block" do
+      called = false
+      @indexer.to_field("title") do |record, accumulator, context|
+        called = true
+
+        assert_kind_of Traject::Indexer::Context, context
+        
+        assert_kind_of Hash, context.clipboard
+        assert_kind_of Hash, context.output_hash
+
+        assert_same record, context.source_record
+        assert_same @indexer.settings, context.settings
+      end
+
+      @indexer.map_record @record
+
+      assert called
+    end
+  end
+  
+end

data/test/indexer/read_write_test.rb

@@ -0,0 +1,47 @@
+require 'test_helper'
+
+# A little Traject Writer that just keeps everything
+# in an array, just added to settings for easy access
+memory_writer_class = Class.new do
+    def initialize(settings)
+      @settings = settings
+      @settings["memory_writer.added"] = []
+    end
+
+    def put(hash)
+      @settings["memory_writer.added"] << hash
+    end
+
+    def close
+      @settings["memory_writer.closed"] = true
+    end
+  end
+
+describe "Traject::Indexer#process" do 
+  before do
+    @indexer = Traject::Indexer.new
+    @indexer.writer_class = memory_writer_class
+    @file = File.open(support_file_path "test_data.utf8.mrc")
+  end
+
+  it "works" do
+    @indexer.to_field("title") do |record, accumulator, context|
+      accumulator << "ADDED TITLE"
+      assert_equal "title", context.field_name
+    end
+
+    @indexer.process( @file )
+
+    assert @indexer.settings["memory_writer.added"]
+    assert_equal 30, @indexer.settings["memory_writer.added"].length
+    assert_kind_of Hash, @indexer.settings["memory_writer.added"].first
+    assert_equal ["ADDED TITLE"], @indexer.settings["memory_writer.added"].first["title"]
+
+    assert @indexer.settings["memory_writer.closed"]
+
+  end
+
+  
+
+
+end

data/test/indexer/settings_test.rb

@@ -0,0 +1,65 @@
+require 'test_helper'
+
+describe "Traject::Indexer#settings" do 
+  before do
+    @indexer = Traject::Indexer.new
+  end
+
+  it "starts out default hash" do
+    assert_kind_of Hash, @indexer.settings
+    assert_equal Traject::Indexer.default_settings, @indexer.settings
+  end
+
+  it "can take argument to set" do
+    @indexer.settings("foo" => "foo", "bar" => "bar")
+
+    assert_equal "foo", @indexer.settings["foo"]
+    assert_equal "bar", @indexer.settings["bar"]
+  end
+
+  it "has settings DSL to set" do
+    @indexer.instance_eval do
+      settings do
+        store "foo", "foo"
+      end
+    end
+
+    assert_equal "foo", @indexer.settings["foo"]
+  end
+
+  it "merges new values, not completely replaces" do
+    @indexer.settings("one" => "original", "two" => "original", "three" => "original", "four" => "original")
+
+    @indexer.settings do
+      store "two", "second"
+      store "three", "second"
+    end
+
+    @indexer.settings do
+      store "three", "third"
+    end
+
+    @indexer.settings("four" => "fourth")
+
+    {"one" => "original", "two" => "second", "three" => "third", "four" => "fourth"}.each_pair do |key, value|
+      assert_equal value, @indexer.settings[key] 
+    end
+  end
+
+  it "is indifferent between string and symbol" do
+    @indexer.settings[:foo] = "foo 1"
+    @indexer.settings["foo"] = "foo 2"
+
+    assert_equal "foo 2", @indexer.settings[:foo]
+
+    @indexer.settings do
+      store "foo", "foo 3"
+      store :foo, "foo 4"
+    end
+
+    assert_equal "foo 4", @indexer.settings["foo"]
+  end
+
+
+
+end

data/test/marc_extractor_test.rb

@@ -0,0 +1,168 @@
+# encoding: UTF-8
+
+require 'test_helper'
+require 'traject/marc_extractor'
+
+
+describe "Traject::MarcExtractor" do
+  describe "#parse_marc_spec" do
+    it "parses single spec with all elements" do
+      parsed = Traject::MarcExtractor.parse_string_spec("245|1*|abcg")
+
+      assert_kind_of Hash, parsed
+      assert_equal 1, parsed.keys.length
+      assert_kind_of Hash, parsed["245"]
+
+      assert_kind_of Array, parsed["245"][:indicators]
+      assert_equal 2, parsed["245"][:indicators].length
+      assert_equal "1", parsed["245"][:indicators][0]
+      assert_nil parsed["245"][:indicators][1]
+
+      assert_kind_of Array, parsed["245"][:subfields]
+
+    end
+
+    it "parses a mixed bag" do
+      parsed = Traject::MarcExtractor.parse_string_spec("245abcde:810:700|*4|bcd")
+
+      assert_length 3, parsed
+
+      #245abcde
+      assert parsed["245"]
+      assert_nil parsed["245"][:indicators]
+      assert_equal %w{a b c d e}, parsed["245"][:subfields]
+
+      #810
+      assert parsed["810"]
+      assert_nil parsed["810"][:indicators]
+      assert_nil parsed["810"][:subfields]
+
+      #700-*4bcd
+      assert parsed["700"]
+      assert_equal [nil, "4"], parsed["700"][:indicators]
+      assert_equal %w{b c d}, parsed["700"][:subfields]
+    end
+
+    it "parses fixed field byte offsets" do
+      parsed = Traject::MarcExtractor.parse_string_spec("005[5]:008[7-10]")
+
+      assert_equal 5, parsed["005"][:bytes]
+      assert_equal 7..10, parsed["008"][:bytes]
+    end
+  end
+
+  describe "#extract_by_spec" do
+    before do
+      @record = MARC::Reader.new(support_file_path  "manufacturing_consent.marc").to_a.first
+    end
+
+    describe "extracts a basic case" do
+      before do
+        parsed_spec = Traject::MarcExtractor.parse_string_spec("700abcdef:856|*2|:505|1*|:245ba")
+        @values = Traject::MarcExtractor.extract_by_spec(@record, parsed_spec)
+      end
+
+      it "returns an array" do
+        assert_kind_of Array, @values
+      end
+
+      it "handles no subfields given" do
+        a856s = @record.find_all {|f| f.tag == "856"}
+        assert a856s, "Record must have 856 fields for this test to work"
+
+        a856s.each do |field|
+          assert @values.include?( field.subfields.collect(&:value).join(" "))
+        end
+      end
+
+      it "does not have 505, due to non-matching indicators" do
+        assert ! @values.find {|s| s.include? "propaganda model"}
+      end
+
+
+
+      it "respects original record order, for both fields and subfields" do
+        expected = ["Manufacturing consent : the political economy of the mass media /",
+                    "Chomsky, Noam.",
+                    "Contributor biographical information http://www.loc.gov/catdir/bios/random051/2001050014.html",
+                    "Publisher description http://www.loc.gov/catdir/description/random044/2001050014.html"]
+        assert_equal expected, @values
+      end
+    end
+
+    describe "extracts fixed fields" do
+      it ", complete" do
+        parsed_spec = Traject::MarcExtractor.parse_string_spec("001")
+        values = Traject::MarcExtractor.extract_by_spec(@record, parsed_spec)
+
+        assert_equal ["2710183"], values
+      end
+      it ", single byte offset" do
+        parsed_spec = Traject::MarcExtractor.parse_string_spec("008[5]")
+        values = Traject::MarcExtractor.extract_by_spec(@record, parsed_spec)
+
+        assert_equal ["1"], values
+      end
+      it ", byte range" do
+        parsed_spec = Traject::MarcExtractor.parse_string_spec("008[7-10]")
+        values = Traject::MarcExtractor.extract_by_spec(@record, parsed_spec)
+
+        assert_equal ["2002"], values
+      end
+    end
+
+    describe "seperator argument" do
+      it "causes non-join when nil" do
+        parsed_spec = Traject::MarcExtractor.parse_string_spec("245")
+        values = Traject::MarcExtractor.extract_by_spec(@record, parsed_spec, :seperator => nil)
+
+        assert_length 3, values
+      end
+
+      it "can be non-default" do
+        parsed_spec = Traject::MarcExtractor.parse_string_spec("245")
+        values = Traject::MarcExtractor.extract_by_spec(@record, parsed_spec, :seperator => "!! ")
+
+        assert_length 1, values
+        assert_equal "Manufacturing consent :!! the political economy of the mass media /!! Edward S. Herman and Noam Chomsky ; with a new introduction by the authors.", values.first
+      end
+    end
+
+    describe "extracts alternate script" do
+      before do
+        @record = MARC::Reader.new(support_file_path  "hebrew880s.marc").to_a.first
+        @parsed_spec = Traject::MarcExtractor.parse_string_spec("245b")
+      end
+      it "from default :include" do
+
+        values = Traject::MarcExtractor.extract_by_spec(@record, @parsed_spec)
+
+        assert_length 2, values # both the original and the 880
+        assert_equal ["ben Marṭin Buber le-Aharon Daṿid Gordon /", "בין מרטין בובר לאהרן דוד גורדון /"], values
+      end
+      it "with :only" do
+        values = Traject::MarcExtractor.extract_by_spec(@record, @parsed_spec, :alternate_script => :only)
+
+        assert_length 1, values
+        assert_equal ["בין מרטין בובר לאהרן דוד גורדון /"], values
+      end
+      it "with false" do
+        values = Traject::MarcExtractor.extract_by_spec(@record, @parsed_spec, :alternate_script => false)
+
+        assert_length 1, values
+        assert_equal ["ben Marṭin Buber le-Aharon Daṿid Gordon /"], values
+      end
+    end
+
+    it "works with string second arg too" do
+      values = Traject::MarcExtractor.extract_by_spec(@record, "245abc")
+
+      assert_length 1, values
+      assert values.first.include?("Manufacturing consent"), "Extracted value includes title"
+    end
+
+  end
+
+
+
+end