marc 1.3.0 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: faaea90f6a961fd419d41cd3fc789a7967ddd41866b82cb2075be3a139e73e58
-  data.tar.gz: dacb3e74659b39f4774205231c07a09802af24bf15726cd8eecee1c424199a56
+  metadata.gz: 8e34cbc11eeea3ea9745181b1f3a958430842993c3f7d3624d1251eb3e07d34e
+  data.tar.gz: f10be646a7e983ebed7ac2e27e2d7733335d5fae66ec2c556d694e49d2e4bba0
 SHA512:
-  metadata.gz: c5860c18fac9062dc15cea515e925a31e652bb599ddb5ef96281ac1840860b607a0d5c2bc66e514ab7d0ea17d1f08219253f54eeff128a9d6251a73627a4a3ce
-  data.tar.gz: 3936a4dc7fd88038e9794cec0427e0f0ee1264639f700d7ae8cf37c9710bc7ff4bba0c7679c8c301034ca3ea98c36e6c5df81c77caff8b6dfe2a9cdf6928dcaf
+  metadata.gz: d86a24aa80866e8bde4579615909d5eef1a753b8ae483e9a9928ae1d1417a1d7819ac87339e0d2fd5a5ac9b220370790aacfb365805f7790c80e54a9e23cdcc0
+  data.tar.gz: 396870e7768e6342a7d7d9950fbef801a89e5219284217a20121f62b1e641395dfaded36b3db903e957da7751fa755221177ed4301bafcd41cbd688fbd858a55

data/.github/workflows/ruby.yml

@@ -1,6 +1,6 @@
 name: CI
 
-on: [push, pull_request]
+on: [push, pull_request, workflow_dispatch]
 
 env:
   # See https://github.com/jruby/jruby/issues/5509
@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        ruby: [2.7, 3.0, 3.1, 3.2, 3.3, 3.4, jruby, truffleruby, "truffleruby+graalvm"]
+        ruby: [2.4, 2.7, 3.0, 3.1, 3.2, 3.3, 3.4, 4.0, jruby, truffleruby, "truffleruby+graalvm"]
     steps:
     - uses: actions/checkout@v4
     - name: Set up Ruby
@@ -19,6 +19,6 @@ jobs:
       with:
         ruby-version: ${{ matrix.ruby }}
     - name: Install dependencies
-      run: bundle install --without documentation
+      run: bundle install
     - name: Run tests
       run: bundle exec rake

data/CHANGELOG.md

@@ -2,12 +2,27 @@
 
 All notable changes to this project will be documented in this file.
 
-## [1.3] - 2025-01-0-9
+## [1.4] - 2026-01-22
 
 ### Breaking Change
 
-**ruby >= 2.2 is now required**. Removed no-longer-necessary `unf` gem in 
-favor of built-in string methods for dealing with encodings. 
+**ruby >= 2.4 is now required**.
+
+2.3 was EOL'd almost 7 years ago, and backporting support for other
+gems was deemed to be not worth it.
+
+This is mostly a cleanup release, to squash warnings
+about frozen strings in more recent rubys, update the
+CI configuration.
+
+## [1.3] - 2025-01-09
+
+### Breaking Change
+
+**ruby >= 2.2 is now required**. 
+
+- Removed no-longer-necessary `unf` gem in favor of built-in string 
+  methods for dealing with encodings. (Aaron Elkiss)
 
 ### Non-user-facing changes
 

data/lib/marc/controlfield.rb

@@ -27,7 +27,7 @@ module MARC
     # The constructor which must be passed a tag value and
     # an optional value for the field.
 
-    def initialize(tag, value = "")
+    def initialize(tag, value = +"")
       @tag = tag
       @value = value
     end

data/lib/marc/dublincore.rb

@@ -26,39 +26,41 @@ module MARC
       end
 
       dc_hash["publisher"] = begin
-        get_field_value(record["260"]["a"]["b"])
-      rescue
-        nil
-      end
+                               get_field_value(record["260"]["a"]["b"])
+                             rescue
+                               nil
+                             end
       dc_hash["date"] = begin
-        get_field_value(record["260"]["c"])
-      rescue
-        nil
-      end
+                          get_field_value(record["260"]["c"])
+                        rescue
+                          nil
+                        end
       dc_hash["type"] = get_field_value(record["655"])
       dc_hash["format"] = begin
-        get_field_value(record["856"]["q"])
-      rescue
-        nil
-      end
+                            get_field_value(record["856"]["q"])
+                          rescue
+                            nil
+                          end
       dc_hash["identifier"] = begin
-        get_field_value(record["856"]["u"])
-      rescue
-        nil
-      end
+                                get_field_value(record["856"]["u"])
+                              rescue
+                                nil
+                              end
       dc_hash["source"] = begin
-        get_field_value(record["786"]["o"]["t"])
-      rescue
-        nil
-      end
+                            get_field_value(record["786"]["o"]["t"])
+                          rescue
+                            nil
+                          end
       dc_hash["language"] = get_field_value(record["546"])
 
       dc_hash["relation"] = []
       dc_hash["relation"] << get_field_value(record["530"])
       ("760".."787").each do |field|
-        dc_hash["relation"] << get_field_value(record[field]["o"]["t"])
-      rescue
-        nil
+        begin
+          dc_hash["relation"] << get_field_value(record[field]["o"]["t"])
+        rescue
+          nil
+        end
       end
 
       ["651", "752"].each do |field|

data/lib/marc/jsonl_writer.rb

@@ -33,7 +33,7 @@ module MARC
     # @param [MARC::Record] record
     # @return [String] MARC-in-JSON representation of the record
     def self.encode(record)
-      JSON.fast_generate(record.to_hash)
+      JSON.generate(record.to_hash)
     end
 
     # @see MARC::JSONLWriter.encode

data/lib/marc/marc8/to_unicode.rb

@@ -65,14 +65,21 @@ module MARC
         normalization = options.fetch(:normalization, :nfc)
 
         # don't choke on empty marc8_string
-        return "" if marc8_string.nil? || marc8_string.empty?
+        return +"" if marc8_string.nil? || marc8_string.empty?
 
         # Make sure to call it 'binary', so we can slice it
         # byte by byte, and so ruby doesn't complain about bad
         # bytes for some other encoding. Yeah, we're changing
         # encoding on input! If it's Marc8, it ought to be tagged
         # binary already.
-        marc8_string.force_encoding("binary")
+
+        # Due to the changes with default frozen strings, we'll check
+        # to see if it's binary already, and only duplicate the string
+        # if it's not. It SHOULD be binary already.
+        unless marc8_string.encoding.to_s == "ASCII-8BIT"
+          marc8_string = marc8_string.dup
+          marc8_string.force_encoding("binary")
+        end
 
         uni_list = []
         combinings = []
@@ -130,7 +137,7 @@ module MARC
           end
 
           if (code_point < 0x20) ||
-              ((code_point > 0x80) && (code_point < 0xa0))
+             ((code_point > 0x80) && (code_point < 0xa0))
             uni = unichr(code_point)
             next
           end

data/lib/marc/reader.rb

@@ -301,7 +301,11 @@ module MARC
       # And now that we've recorded the current encoding, we force
       # to binary encoding, because we're going to be doing byte arithmetic,
       # and want to avoid byte-vs-char confusion.
-      marc.force_encoding("binary") if marc.respond_to?(:force_encoding)
+
+      if (marc.respond_to?(:force_encoding) && marc.encoding != "ASCII-8BIT")
+        marc = marc.dup
+        marc.force_encoding("binary")
+      end
 
       record = Record.new
       record.leader = marc[0..LEADER_LENGTH - 1]
@@ -346,7 +350,7 @@ module MARC
         # if we were told to be forgiving we just use the
         # next available chuck of field data that we
         # split apart based on the END_OF_FIELD
-        field_data = ""
+        field_data = +""
         if params[:forgiving]
           field_data = all_fields.shift
 
@@ -442,10 +446,10 @@ module MARC
         # in future implementations.
         if params[:internal_encoding]
           str = if RUBY_VERSION >= "3.0"
-            str.encode(params[:internal_encoding], **params)
-          else
-            str.encode(params[:internal_encoding], params)
-          end
+                  str.encode(params[:internal_encoding], **params)
+                else
+                  str.encode(params[:internal_encoding], params)
+                end
         elsif params[:invalid] || params[:replace] || (params[:validate_encoding] == true)
 
           if params[:validate_encoding] == true && !str.valid_encoding?
@@ -482,11 +486,13 @@ module MARC
   class ForgivingReader < Reader
     def each
       @handle.each_line(END_OF_RECORD) do |raw|
-        record = MARC::Reader.decode(raw, @encoding_options.merge(forgiving: true))
-        yield record
-      rescue
-        # caught exception just keep barrelling along
-        # TODO add logging
+        begin
+          record = MARC::Reader.decode(raw, @encoding_options.merge(forgiving: true))
+          yield record
+        rescue
+          # caught exception just keep barrelling along
+          # TODO add logging
+        end
       end
     end
   end

data/lib/marc/subfield.rb

@@ -7,11 +7,11 @@ module MARC
   class Subfield
     attr_accessor :code, :value
 
-    def initialize(code = "", value = "")
+    def initialize(code = +"", value = +"")
       # can't allow code of value to be nil
       # or else it'll screw us up later on
-      @code = code.nil? ? "" : code
-      @value = value.nil? ? "" : value
+      @code = code.nil? ? +"" : code
+      @value = value.nil? ? +"" : value
     end
 
     def ==(other)

data/lib/marc/version.rb

@@ -1,3 +1,3 @@
 module MARC
-  VERSION = "1.3.0"
+  VERSION = "1.4.0"
 end

data/lib/marc/writer.rb

@@ -62,12 +62,12 @@ module MARC
     # Second arg allow_oversized, default false, set to true
     # to raise on MARC record that can't fit into ISO 2709.
     def self.encode(record, allow_oversized = false)
-      directory = ""
-      fields = ""
+      directory = +""
+      fields = +""
       offset = 0
       record.each do |field|
         # encode the field
-        field_data = ""
+        field_data = +""
         if field.instance_of?(MARC::DataField)
           warn("Warn:  Missing indicator") unless field.indicator1 && field.indicator2
           field_data = (field.indicator1 || " ") + (field.indicator2 || " ")

data/lib/marc/xml_parsers.rb

@@ -56,7 +56,7 @@ module MARC
     SF_TAG = "subfield".freeze
 
     def init
-      @record = {record: nil, leader: "", field: nil, subfield: nil}
+      @record = {record: nil, leader: +"", field: nil, subfield: nil}
       @current_element = nil
       @ns = "http://www.loc.gov/MARC21/slim"
     end
@@ -115,7 +115,7 @@ module MARC
         when REC_TAG then yield_record
         when LEAD_TAG
           @record[:record].leader = @record[:leader]
-          @record[:leader] = ""
+          @record[:leader] = +""
           @current_element = nil if @current_element == :leader
         end
       end
@@ -238,7 +238,7 @@ module MARC
       data_field = nil
       control_field = nil
       subfield = nil
-      text = ""
+      text = +""
       attrs = nil
       if Module.constants.index("Nokogiri") && @parser.is_a?(Nokogiri::XML::Reader)
         datafield = nil
@@ -295,18 +295,18 @@ module MARC
           end
 
           if event.start_element?
-            text = ""
+            text = +""
             attrs = event[1]
             case strip_ns(event[0])
             when "controlfield"
-              text = ""
+              text = +""
               control_field = MARC::ControlField.new(attrs[TAG])
             when "datafield"
-              text = ""
+              text = +""
               data_field = MARC::DataField.new(attrs[TAG], attrs[IND1],
                 attrs[IND2])
             when "subfield"
-              text = ""
+              text = +""
               subfield = MARC::Subfield.new(attrs[CODE])
             end
           end

data/marc.gemspec

@@ -8,7 +8,7 @@ Gem::Specification.new do |s|
   s.homepage = "https://github.com/ruby-marc/ruby-marc/"
   s.summary = "A ruby library for working with Machine Readable Cataloging"
   s.license = "MIT"
-  s.required_ruby_version = ">= 2.2.0"
+  s.required_ruby_version = ">= 2.3.0"
   s.authors = ["Kevin Clarke", "Bill Dueber", "William Groppe", "Jonathan Rochkind", "Ross Singer", "Ed Summers", "Chris Beer"]
 
   s.files = `git ls-files -z`.split("\x0")
@@ -17,12 +17,13 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   s.add_dependency "nokogiri", "~>1.0"
+  s.add_dependency "rexml" # rexml was unbundled from the stdlib in ruby 3
 
   s.add_development_dependency "rake", "~>13.0"
   s.add_development_dependency "test-unit", "~>3.0"
   s.add_development_dependency "standard", "~>1.0"
   s.add_development_dependency "warning", "~>1.5"
   s.add_development_dependency "xml-simple"
+  s.add_development_dependency "rdoc"
 
-  s.add_dependency "rexml" # rexml was unbundled from the stdlib in ruby 3
 end

data/spec/controlfield_spec.rb

@@ -0,0 +1,52 @@
+require 'spec_helper'
+
+RSpec.describe MARC::ControlField do
+  it "formats a control field correctly" do
+    control = MARC::ControlField.new("005", "foobarbaz")
+    expect(control.to_s).to eq("005 foobarbaz")
+  end
+
+  it "rejects data field as control field" do
+    field = MARC::DataField.new("007")
+    expect(field.valid?).to be(false)
+  end
+
+  it "rejects alpha control field tags" do
+    # can't have a field with a tag < 010
+    field = MARC::ControlField.new("DDD")
+    expect(field.valid?).to be(false)
+  end
+
+  it "supports adding custom control field tags" do
+    MARC::ControlField.control_tags << "FMT"
+    field = MARC::ControlField.new("FMT")
+    expect(field.valid?).to be(true)
+    field = MARC::DataField.new("FMT")
+    expect(field.valid?).to be(false)
+    MARC::ControlField.control_tags.delete("FMT")
+    field = MARC::DataField.new("FMT")
+    expect(field.valid?).to be(true)
+    field = MARC::ControlField.new("FMT")
+    expect(field.valid?).to be(false)
+  end
+
+  it "rejects control field with data field tag" do
+    # can't have a control with a tag > 009
+    f = MARC::ControlField.new("245")
+    expect(f.valid?).to be(false)
+  end
+
+  it "compares control fields correctly" do
+    f1 = MARC::ControlField.new("001", "foobarbaz")
+    f2 = MARC::ControlField.new("001", "foobarbaz")
+    expect(f1).to eq(f2)
+
+    f3 = MARC::ControlField.new("001", "foobarbazqux")
+    expect(f1).not_to eq(f3)
+    f4 = MARC::ControlField.new("002", "foobarbaz")
+    expect(f1).not_to eq(f4)
+
+    expect(f1).not_to eq("001")
+    expect(f2).not_to eq("foobarbaz")
+  end
+end

data/spec/datafield_spec.rb

@@ -0,0 +1,75 @@
+require 'spec_helper'
+
+RSpec.describe MARC::DataField do
+  it "handles tags correctly" do
+    f1 = MARC::DataField.new("100")
+    expect(f1.tag).to eq("100")
+    f2 = MARC::DataField.new("100")
+    expect(f2.tag).to eq("100")
+    expect(f1).to eq(f2)
+    f3 = MARC::DataField.new("245")
+    expect(f1).not_to eq(f3)
+  end
+
+  it "handles alphabetic tags" do
+    alph = MARC::DataField.new("ALF")
+    expect(alph.tag).to eq("ALF")
+
+    alphnum = MARC::DataField.new("0D9")
+    expect(alphnum.tag).to eq("0D9")
+  end
+
+  it "handles indicators" do
+    f1 = MARC::DataField.new("100", "0", "1")
+    expect(f1.indicator1).to eq("0")
+    expect(f1.indicator2).to eq("1")
+    f2 = MARC::DataField.new("100", "0", "1")
+    expect(f2.indicator1).to eq("0")
+    expect(f2.indicator2).to eq("1")
+    expect(f1).to eq(f2)
+    f3 = MARC::DataField.new("100", "1", "1")
+    expect(f1).not_to eq(f3)
+  end
+
+  it "handles subfields" do
+    f1 = MARC::DataField.new("100", "0", "1",
+      MARC::Subfield.new("a", "Foo"),
+      MARC::Subfield.new("b", "Bar"))
+    expect(f1.to_s).to eq("100 01 $a Foo $b Bar ")
+    expect(f1.value).to eq("FooBar")
+    f2 = MARC::DataField.new("100", "0", "1",
+      MARC::Subfield.new("a", "Foo"),
+      MARC::Subfield.new("b", "Bar"))
+    expect(f1).to eq(f2)
+    f3 = MARC::DataField.new("100", "0", "1",
+      MARC::Subfield.new("a", "Foo"),
+      MARC::Subfield.new("b", "Bez"))
+    expect(f1).not_to eq(f3)
+  end
+
+  it "supports subfield shorthand" do
+    f = MARC::DataField.new("100", "0", "1", ["a", "Foo"], ["b", "Bar"])
+    expect(f.to_s).to eq("100 01 $a Foo $b Bar ")
+  end
+
+  it "iterates through subfields" do
+    field = MARC::DataField.new("100", "0", "1", ["a", "Foo"], ["b", "Bar"],
+      ["a", "Bez"])
+    count = 0
+    field.each { |x| count += 1 }
+    expect(count).to eq(3)
+  end
+
+  it "supports lookup shorthand" do
+    f = MARC::DataField.new("100", "0", "1", ["a", "Foo"], ["b", "Bar"])
+    expect(f["b"]).to eq("Bar")
+  end
+
+  it "distinguishes from other types" do
+    f = MARC::DataField.new("100", "0", "1",
+      MARC::Subfield.new("a", "Foo"),
+      MARC::Subfield.new("b", "Bar"))
+    expect(f).not_to eq("100 01 $a Foo $b Bar ")
+    expect(f).not_to eq(f["a"])
+  end
+end

data/spec/reader_char_encodings_spec.rb

@@ -0,0 +1,245 @@
+require 'spec_helper'
+require 'stringio'
+
+# Testing char encodings under 1.9, don't bother running
+# these tests except under 1.9, will either fail (because
+# 1.9 func the test itself uses isn't there), or trivially pass
+# (because the func they are testing is no-op on 1.9).
+
+if "".respond_to?(:encoding)
+  RSpec.describe "Reader Character Encodings" do
+    # Common test files
+    let(:utf_marc_path) { "test/utf8.marc" }
+    let(:cp866_marc_path) { "test/cp866_multirecord.marc" }
+    let(:bad_marc8_path) { "test/bad_eacc_encoding.marc8.marc" }
+    
+    # Helper methods
+    def assert_utf8_right_in_utf8(record)
+      expect(record["245"].subfields.first.value.encoding.name).to eq("UTF-8")
+      expect(record["245"].to_s.encoding.name).to eq("UTF-8")
+      expect(record["245"].subfields.first.to_s.encoding.name).to eq("UTF-8")
+      expect(record["245"].subfields.first.value.encoding.name).to eq("UTF-8")
+      expect(record["245"]["a"].encoding.name).to eq("UTF-8")
+      expect(record["245"]["a"]).to start_with("Photčhanānukrom")
+    end
+
+    def assert_cp866_right(record, encoding = "IBM866")
+      expect(record["001"].value.encoding.name).to eq(encoding)
+      expect(record["001"].value.encode("UTF-8").unpack("H4")).to eq(["d09d"]) # russian capital N
+    end
+
+    def assert_all_values_valid_encoding(record, encoding_name = "UTF-8")
+      record.fields.each do |field|
+        if field.is_a? MARC::DataField
+          field.subfields.each do |sf|
+            expect(sf.value.encoding.name).to eq(encoding_name), "Is tagged #{encoding_name}: #{field.tag}: #{sf}"
+            expect(field.value.valid_encoding?).to be(true), "Is valid encoding: #{field.tag}: #{sf}"
+          end
+        else
+          expect(field.value.encoding.name).to eq(encoding_name), "Is tagged #{encoding_name}: #{field}"
+          expect(field.value.valid_encoding?).to be(true), "Is valid encoding: #{field}"
+        end
+      end
+    end
+
+    it "loads unicode correctly" do
+      reader = MARC::Reader.new(utf_marc_path)
+      record = nil
+      expect { record = reader.first }.not_to raise_error
+      assert_utf8_right_in_utf8(record)
+    end
+
+    it "decodes unicode with forgiving mode" do
+      # two kinds of forgiving invocation, they shouldn't be different,
+      # but just in case they have slightly different code paths, test em too.
+      marc_string = File.read(utf_marc_path).force_encoding("utf-8")
+      record = MARC::Reader.decode(marc_string, forgiving: true)
+      assert_utf8_right_in_utf8(record)
+
+      reader = MARC::ForgivingReader.new(utf_marc_path)
+      record = reader.first
+      assert_utf8_right_in_utf8(record)
+    end
+
+    it "passes options through ForgivingReader" do
+      # Make sure ForgivingReader accepts same options as MARC::Reader
+      # We don't test them ALL though, just a sample.
+      # Tell it we're reading cp866, but trancode to utf8 for us.
+      reader = MARC::ForgivingReader.new(cp866_marc_path, external_encoding: "cp866", internal_encoding: "utf-8")
+      record = reader.first
+      assert_cp866_right(record, "UTF-8")
+    end
+
+    it "handles explicit encoding" do
+      reader = MARC::Reader.new(cp866_marc_path, external_encoding: "cp866")
+      assert_cp866_right(reader.first, "IBM866")
+    end
+
+    it "raises error on bad encoding name" do
+      reader = MARC::Reader.new(cp866_marc_path, external_encoding: "adadfadf")
+      expect { reader.first }.to raise_error(ArgumentError)
+    end
+
+    it "handles marc8 with binary encoding" do
+      # Marc8, if we want to keep it without transcoding, best we can do is read it in binary.
+      reader = MARC::Reader.new("test/marc8_accented_chars.marc", external_encoding: "binary")
+      record = reader.first
+      expect(record["100"].subfields.first.value.encoding.name).to eq("ASCII-8BIT")
+    end
+
+    it "converts marc8 to unicode" do
+      reader = MARC::Reader.new("test/marc8_accented_chars.marc", external_encoding: "MARC-8")
+      record = reader.first
+      assert_all_values_valid_encoding(record)
+      expect(record["100"]["a"]).to eq("Serreau, Geneviève.")
+    end
+
+    it "converts marc8 to unicode with file handle" do
+      # had some trouble with this one, let's ensure it with a test
+      file = File.new("test/marc8_accented_chars.marc")
+      reader = MARC::Reader.new(file, external_encoding: "MARC-8")
+      record = reader.first
+      assert_all_values_valid_encoding(record)
+    end
+
+    it "handles marc8 with character entities" do
+      reader = MARC::Reader.new("test/escaped_character_reference.marc8.marc", external_encoding: "MARC-8")
+      record = reader.first
+      assert_all_values_valid_encoding(record)
+      expect(record["260"]["a"]).to eq("Rio de Janeiro escaped replacement char: \uFFFD .")
+    end
+
+    it "raises error on bad marc8" do
+      expect {
+        reader = MARC::Reader.new(bad_marc8_path, external_encoding: "MARC-8")
+        reader.first
+      }.to raise_error(Encoding::InvalidByteSequenceError)
+    end
+
+    it "handles bad marc8 with replacement" do
+      reader = MARC::Reader.new(bad_marc8_path, external_encoding: "MARC-8", invalid: :replace, replace: "[?]")
+      record = reader.first
+      assert_all_values_valid_encoding(record)
+      expect(record["880"]["a"]).to include("[?]")
+    end
+
+    it "handles files opened with external encoding" do
+      reader = MARC::Reader.new(File.open(cp866_marc_path, "r:cp866"))
+      record = reader.first
+      assert_cp866_right(record, "IBM866")
+    end
+
+    it "prioritizes explicit encoding over file encoding" do
+      reader = MARC::Reader.new(File.open(cp866_marc_path, "r:utf-8"), external_encoding: "cp866")
+      record = reader.first
+      assert_cp866_right(record, "IBM866")
+    end
+
+    it "handles strings with utf8 encoding" do
+      marc_file = File.open(utf_marc_path)
+      reader = MARC::Reader.new(marc_file)
+      expect { reader.first }.not_to raise_error
+    end
+
+    it "handles utf8 with bad bytes" do
+      marc_file = File.open("test/marc_with_bad_utf8.utf8.marc")
+      reader = MARC::Reader.new(marc_file, invalid: :replace)
+      record = reader.first
+
+      record.fields.each do |field|
+        if field.is_a? MARC::ControlField
+          expect(field.value.encoding.name).to eq("UTF-8")
+          expect(field.value.valid_encoding?).to be(true)
+        else
+          field.subfields.each do |subfield|
+            expect(subfield.value.encoding.name).to eq("UTF-8")
+            expect(subfield.value.valid_encoding?).to be(true)
+          end
+        end
+      end
+
+      expect(record["520"]["a"]).to include("\uFFFD")
+    end
+
+    it "handles string with cp866 encoding" do
+      marc_string = File.read(cp866_marc_path).force_encoding("cp866")
+      reader = MARC::Reader.new(StringIO.new(marc_string))
+      record = reader.first
+      assert_cp866_right(record, "IBM866")
+    end
+
+    it "decodes strings with cp866 encoding" do
+      marc_string = File.read(cp866_marc_path).force_encoding("cp866")
+      record = MARC::Reader.decode(marc_string)
+      assert_cp866_right(record, "IBM866")
+    end
+
+    it "supports transcoding" do
+      reader = MARC::Reader.new(cp866_marc_path,
+        external_encoding: "cp866",
+        internal_encoding: "UTF-8")
+      record = reader.first
+      assert_cp866_right(record, "UTF-8")
+    end
+
+    it "works with binary filehandle" do
+      # about to recommend this as a foolproof way to avoid
+      # ruby transcoding behind your back in docs, let's make
+      # sure it really works.
+      reader = MARC::Reader.new(File.open(cp866_marc_path, external_encoding: "binary", internal_encoding: "binary"),
+        external_encoding: "IBM866")
+      record = reader.first
+      assert_cp866_right(record, "IBM866")
+    end
+
+    it "handles bad source bytes" do
+      reader = MARC::Reader.new("test/utf8_with_bad_bytes.marc",
+        external_encoding: "UTF-8",
+        validate_encoding: true)
+      expect { reader.first }.to raise_error(Encoding::InvalidByteSequenceError)
+    end
+
+    it "replaces bad source bytes when configured" do
+      reader = MARC::Reader.new("test/utf8_with_bad_bytes.marc",
+        external_encoding: "UTF-8", invalid: :replace)
+      record = nil
+      expect { record = reader.first }.not_to raise_error
+      expect(record["245"]["a"]).to match(/=> #{"\uFFFD"} \(<=/)
+    end
+
+    it "supports custom replacement for bad bytes" do
+      reader = MARC::Reader.new("test/utf8_with_bad_bytes.marc",
+        external_encoding: "UTF-8", invalid: :replace, replace: "")
+      record = reader.first
+      expect(record["245"]["a"]).to match(/=> \( <=/)
+    end
+
+    it "works with default_internal encoding" do
+      original = Encoding.default_internal
+      Encoding.default_internal = "UTF-8"
+
+      reader = MARC::Reader.new(File.open(cp866_marc_path, "r:cp866"))
+      record = reader.first
+      assert_cp866_right(record, "IBM866")
+    ensure
+      Encoding.default_internal = original
+    end
+
+    it "works with default_internal encoding using string arg" do
+      original = Encoding.default_internal
+      Encoding.default_internal = "UTF-8"
+
+      reader = MARC::Reader.new(cp866_marc_path, external_encoding: "cp866")
+      record = reader.first
+      assert_cp866_right(record, "IBM866")
+    ensure
+      Encoding.default_internal = original
+    end
+  end
+else
+  RSpec.describe "Reader Character Encodings" do
+    it "skips tests on Ruby < 1.9" do
+      skip("Tests not being run in ruby 1.9.x or higher")
+    end
+  end
+end

data/spec/reader_spec.rb

@@ -0,0 +1,108 @@
+require 'spec_helper'
+
+RSpec.describe MARC::Reader do
+  it "reads batch records correctly" do
+    reader = MARC::Reader.new("test/batch.dat")
+    count = 0
+    reader.each { count += 1 }
+    expect(count).to eq(10)
+  end
+
+  it "handles loose records with ForgivingReader" do
+    reader = MARC::ForgivingReader.new("test/batch.dat")
+    count = 0
+    reader.each { count += 1 }
+    expect(count).to eq(10)
+  end
+
+  it "handles UTF-8 in ForgivingReader" do
+    # This isn't actually a corrupt file, but it is utf8,
+    # and I have some reason to believe forgiving reader isn't
+    # working properly with UTF8 in ruby 1.9, so testing it.
+    reader = MARC::ForgivingReader.new("test/utf8.marc")
+    count = 0
+    reader.each { count += 1 }
+    expect(count).to eq(1)
+  end
+
+  it "handles unimarc records" do
+    # Unimarc might use a different record seperator? Let's make sure it works.
+    reader = MARC::Reader.new(File.open("test/cp866_unimarc.marc", "r:cp866"))
+    count = 0
+    reader.each { |a| count += 1 }
+    expect(count).to eq(1)
+  end
+
+  it "handles non-numeric tags" do
+    reader = MARC::Reader.new("test/non-numeric.dat")
+    count = 0
+    record = nil
+    reader.each do |rec|
+      count += 1
+      record = rec
+    end
+    expect(count).to eq(1)
+    expect(record["ISB"]["a"]).to eq("9780061317842")
+    expect(record["LOC"]["9"]).to eq("1")
+  end
+
+  it "raises exception for bad MARC data" do
+    reader = MARC::Reader.new("test/tc_reader.rb")
+    expect { reader.entries[0] }.to raise_error(MARC::Exception)
+  end
+
+  it "supports search functionality" do
+    reader = MARC::Reader.new("test/batch.dat")
+    records = reader.find_all { |r| r =~ /Perl/ }
+    expect(records.length).to eq(10)
+
+    reader = MARC::Reader.new("test/batch.dat")
+    records = reader.find_all { |r| r["245"] =~ /Perl/ }
+    expect(records.length).to eq(10)
+
+    reader = MARC::Reader.new("test/batch.dat")
+    records = reader.find_all { |r| r["245"]["a"] =~ /Perl/ }
+    expect(records.length).to eq(10)
+
+    reader = MARC::Reader.new("test/batch.dat")
+    records = reader.find_all { |r| r =~ /Foo/ }
+    expect(records.length).to eq(0)
+  end
+
+  it "provides a binary enumerator" do
+    reader = MARC::Reader.new("test/batch.dat")
+    iter = reader.each
+    r = iter.next
+    expect(r).to be_an_instance_of(MARC::Record)
+    9.times { iter.next } # total of ten records
+    expect { iter.next }.to raise_error(StopIteration)
+  end
+
+  it "supports each_raw method" do
+    reader = MARC::Reader.new("test/batch.dat")
+    count = 0
+    raw = nil
+    reader.each_raw { |r|
+      count += 1
+      raw = r
+    }
+    expect(count).to eq(10)
+    expect(raw).to be_an_instance_of(String)
+
+    record = MARC::Reader.decode(raw)
+    expect(record).to be_an_instance_of(MARC::Record)
+  end
+
+  it "supports each_raw enumerator" do
+    reader = MARC::Reader.new("test/batch.dat")
+    enum = reader.each_raw
+    r = enum.next
+    expect(r).to be_an_instance_of(String)
+
+    record = MARC::Reader.decode(r)
+    expect(record).to be_an_instance_of(MARC::Record)
+
+    9.times { enum.next } # total of ten records
+    expect { enum.next }.to raise_error(StopIteration)
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,24 @@
+require 'rspec'
+require 'marc'
+
+RSpec.configure do |config|
+  config.expect_with :rspec do |expectations|
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  config.mock_with :rspec do |mocks|
+    mocks.verify_partial_doubles = true
+  end
+
+  config.shared_context_metadata_behavior = :apply_to_host_groups
+  config.filter_run_when_matching :focus
+  config.disable_monkey_patching!
+  config.warnings = true
+
+  if config.files_to_run.one?
+    config.default_formatter = "doc"
+  end
+
+  config.order = :random
+  Kernel.srand config.seed
+end

data/spec/writer_spec.rb

@@ -0,0 +1,121 @@
+require 'spec_helper'
+require 'stringio'
+
+RSpec.describe MARC::Writer do
+  it "writes and reads MARC records properly" do
+    writer = MARC::Writer.new("test/writer.dat")
+    record = MARC::Record.new
+    record.append(MARC::DataField.new("245", "0", "1", ["a", "foo"]))
+    writer.write(record)
+    writer.close
+
+    # read it back to make sure
+    reader = MARC::Reader.new("test/writer.dat")
+    records = reader.entries
+    expect(records.length).to eq(1)
+    expect(records[0]).to eq(record)
+
+    # cleanup
+    File.unlink("test/writer.dat")
+  end
+
+  if "".respond_to?(:encoding)
+    it "handles mixed encodings properly" do
+      writer = MARC::Writer.new("test/writer.dat")
+
+      # MARC::Writer should just happily write out whatever bytes you give it, even
+      # mixing encodings that can't be mixed. We ran into an actual example mixing
+      # MARC8 (tagged ruby binary) and UTF8, we want it to be written out.
+
+      record = MARC::Record.new
+
+      record.append MARC::DataField.new("700", "0", " ", ["a", "Nhouy Abhay,".force_encoding("BINARY")], ["c", "Th\xE5ao,".force_encoding("BINARY")], ["d", "1909-"])
+      record.append MARC::DataField.new("700", "0", " ", ["a", "Somchin P\xF8\xE5o. Ngin,".force_encoding("BINARY")])
+
+      record.append MARC::DataField.new("100", "0", "0", ["a", "\xE5angkham. ".force_encoding("BINARY")])
+      record.append MARC::DataField.new("245", "1", "0", ["b", "chef-d'oeuvre de la litt\xE2erature lao".force_encoding("BINARY")])
+
+      # One in UTF8 and marked
+      record.append MARC::DataField.new("999", "0", "1", ["a", "chef-d'ocuvre de la littU+FFC3\U+FFA9rature".force_encoding("UTF-8")])
+
+      writer.write(record)
+      writer.close
+    ensure
+      File.unlink("test/writer.dat") if File.exist?("test/writer.dat")
+    end
+  end
+
+  it "supports oversized records when configured" do
+    too_long_record = MARC::Record.new
+    1.upto(1001) do
+      too_long_record.append MARC::DataField.new("500", " ", " ", ["a", "A really long record.1234567890123456789012345678901234567890123456789012345678901234567890123456789"])
+    end
+
+    wbuffer = StringIO.new("", "w")
+    writer = MARC::Writer.new(wbuffer)
+    writer.allow_oversized = true
+
+    writer.write(too_long_record)
+    writer.close
+
+    expect(wbuffer.string.slice(0, 5)).to eq("00000")
+
+    rbuffer = StringIO.new(wbuffer.string.dup)
+
+    # Forgiving reader will, round trippable
+    new_record = MARC::Reader.decode(rbuffer.string, forgiving: true)
+    expect(new_record).to eq(too_long_record)
+
+    # Test in the middle of a MARC file
+    good_record = MARC::Record.new
+    good_record.append MARC::DataField.new("500", " ", " ", ["a", "A short record"])
+    wbuffer = StringIO.new("", "w")
+    writer = MARC::Writer.new(wbuffer)
+    writer.allow_oversized = true
+
+    writer.write(good_record)
+    writer.write(too_long_record)
+    writer.write(good_record)
+
+    rbuffer = StringIO.new(wbuffer.string.dup)
+    reader = MARC::ForgivingReader.new(rbuffer)
+    records = reader.to_a
+
+    expect(records.length).to eq(3)
+    expect(records[0]).to eq(good_record)
+    expect(records[2]).to eq(good_record)
+    expect(records[1]).to eq(too_long_record)
+  end
+
+  it "raises exception for oversized records by default" do
+    too_long_record = MARC::Record.new
+    1.upto(1001) do
+      too_long_record.append MARC::DataField.new("500", " ", " ", ["a", "A really long record.1234567890123456789012345678901234567890123456789012345678901234567890123456789"])
+    end
+
+    wbuffer = StringIO.new("", "w")
+    writer = MARC::Writer.new(wbuffer)
+
+    expect { writer.write too_long_record }.to raise_error(MARC::Exception)
+  end
+
+  it "handles forgiving writing" do
+    marc = "00305cam a2200133 a 4500001000700000003000900007005001700016008004100033008004100074035002500115245001700140909001000157909000400167\036635145\036UK-BiLMS\03620060329173705.0\036s1982iieng6                  000 0 eng||\036060116|||||||||xxk                 eng||\036  \037a(UK-BiLMS)M0017366ZW\03600\037aTest record.\036  \037aa\037b\037c\036\037b0\036\035\000"
+    rec = MARC::Record.new_from_marc(marc)
+    expect { rec.to_marc }.not_to raise_error
+  end
+
+  it "handles Unicode roundtrip" do
+    record = MARC::Reader.new("test/utf8.marc", external_encoding: "UTF-8").first
+
+    writer = MARC::Writer.new("test/writer.dat")
+    writer.write(record)
+    writer.close
+
+    read_back_record = MARC::Reader.new("test/writer.dat", external_encoding: "UTF-8").first
+
+    # Make sure the one we wrote out then read in again
+    # is the same as the one we read the first time
+    expect(record).to eq(read_back_record)
+  end
+end

data/test/tc_writer.rb

@@ -32,14 +32,14 @@ class WriterTest < Test::Unit::TestCase
 
       record = MARC::Record.new
 
-      record.append MARC::DataField.new("700", "0", " ", ["a", "Nhouy Abhay,".force_encoding("BINARY")], ["c", "Th\xE5ao,".force_encoding("BINARY")], ["d", "1909-"])
-      record.append MARC::DataField.new("700", "0", " ", ["a", "Somchin P\xF8\xE5o. Ngin,".force_encoding("BINARY")])
+      record.append MARC::DataField.new("700", "0", " ", ["a", +"Nhouy Abhay,".dup.force_encoding("BINARY")], ["c", +"Th\xE5ao,".dup.force_encoding("BINARY")], ["d", "1909-"])
+      record.append MARC::DataField.new("700", "0", " ", ["a", +"Somchin P\xF8\xE5o. Ngin,".dup.force_encoding("BINARY")])
 
-      record.append MARC::DataField.new("100", "0", "0", ["a", "\xE5angkham. ".force_encoding("BINARY")])
-      record.append MARC::DataField.new("245", "1", "0", ["b", "chef-d'oeuvre de la litt\xE2erature lao".force_encoding("BINARY")])
+      record.append MARC::DataField.new("100", "0", "0", ["a", +"\xE5angkham. ".dup.force_encoding("BINARY")])
+      record.append MARC::DataField.new("245", "1", "0", ["b", +"chef-d'oeuvre de la litt\xE2erature lao".dup.force_encoding("BINARY")])
 
       # One in UTF8 and marked
-      record.append MARC::DataField.new("999", "0", "1", ["a", "chef-d'ocuvre de la littU+FFC3\U+FFA9rature".force_encoding("UTF-8")])
+      record.append MARC::DataField.new("999", "0", "1", ["a", +"chef-d'ocuvre de la littU+FFC3\U+FFA9rature".dup.force_encoding("UTF-8")])
 
       writer.write(record)
       writer.close
@@ -54,7 +54,7 @@ class WriterTest < Test::Unit::TestCase
       too_long_record.append MARC::DataField.new("500", " ", " ", ["a", "A really long record.1234567890123456789012345678901234567890123456789012345678901234567890123456789"])
     end
 
-    wbuffer = StringIO.new("", "w")
+    wbuffer = StringIO.new(+"", "w")
     writer = MARC::Writer.new(wbuffer)
     writer.allow_oversized = true
 
@@ -78,7 +78,7 @@ class WriterTest < Test::Unit::TestCase
     # Test in the middle of a MARC file
     good_record = MARC::Record.new
     good_record.append MARC::DataField.new("500", " ", " ", ["a", "A short record"])
-    wbuffer = StringIO.new("", "w")
+    wbuffer = StringIO.new(+"", "w")
     writer = MARC::Writer.new(wbuffer)
     writer.allow_oversized = true
 
@@ -102,7 +102,7 @@ class WriterTest < Test::Unit::TestCase
       too_long_record.append MARC::DataField.new("500", " ", " ", ["a", "A really long record.1234567890123456789012345678901234567890123456789012345678901234567890123456789"])
     end
 
-    wbuffer = StringIO.new("", "w")
+    wbuffer = StringIO.new(+"", "w")
     writer = MARC::Writer.new(wbuffer)
 
     assert_raise(MARC::Exception) do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: marc
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.4.0
 platform: ruby
 authors:
 - Kevin Clarke
@@ -11,10 +11,9 @@ authors:
 - Ross Singer
 - Ed Summers
 - Chris Beer
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-01-09 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -30,6 +29,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rexml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -101,20 +114,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rexml
+  name: rdoc
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description:
 email: ehs@pobox.com
 executables:
 - marc
@@ -154,6 +166,12 @@ files:
 - lib/marc/xmlreader.rb
 - lib/marc/xmlwriter.rb
 - marc.gemspec
+- spec/controlfield_spec.rb
+- spec/datafield_spec.rb
+- spec/reader_char_encodings_spec.rb
+- spec/reader_spec.rb
+- spec/spec_helper.rb
+- spec/writer_spec.rb
 - test/bad_eacc_encoding.marc8.marc
 - test/batch.dat
 - test/batch.xml
@@ -198,7 +216,6 @@ homepage: https://github.com/ruby-marc/ruby-marc/
 licenses:
 - MIT
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -206,18 +223,23 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.2.0
+      version: 2.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.22
-signing_key:
+rubygems_version: 4.0.3
 specification_version: 4
 summary: A ruby library for working with Machine Readable Cataloging
 test_files:
+- spec/controlfield_spec.rb
+- spec/datafield_spec.rb
+- spec/reader_char_encodings_spec.rb
+- spec/reader_spec.rb
+- spec/spec_helper.rb
+- spec/writer_spec.rb
 - test/bad_eacc_encoding.marc8.marc
 - test/batch.dat
 - test/batch.xml