berkeley_library-marc 0.2.0

data/lib/berkeley_library/marc/module_info.rb

@@ -0,0 +1,14 @@
+module BerkeleyLibrary
+  module Marc
+    class ModuleInfo
+      NAME = 'berkeley_library-marc'.freeze
+      AUTHOR = 'David Moles'.freeze
+      AUTHOR_EMAIL = 'dmoles@berkeley.edu'.freeze
+      SUMMARY = 'MARC utilities for the UC Berkeley Library'.freeze
+      DESCRIPTION = 'A gem providing MARC-related utility code and extensions to ruby-marc for the UC Berkeley Library'.freeze
+      LICENSE = 'MIT'.freeze
+      VERSION = '0.2.0'.freeze
+      HOMEPAGE = 'https://github.com/BerkeleyLibrary/marc'.freeze
+    end
+  end
+end

data/lib/marc_extensions.rb

@@ -0,0 +1 @@
+Dir.glob(File.expand_path('marc_extensions/*.rb', __dir__)).sort.each(&method(:require))

data/lib/marc_extensions/data_field.rb

@@ -0,0 +1,29 @@
+require 'marc'
+require 'marc_extensions/subfield'
+
+module MARCExtensions
+  module DataFieldExtensions
+    def subfield_codes
+      subfields.map(&:code)
+    end
+
+    def frozen?
+      [tag, indicator1, indicator2, subfields].all?(&:frozen?)
+      subfields.all?(&:frozen?)
+    end
+
+    def freeze
+      [tag, indicator1, indicator2].each(&:freeze)
+      subfields.each(&:freeze)
+      subfields.freeze
+      self
+    end
+  end
+end
+
+module MARC
+  # @see https://rubydoc.info/gems/marc/MARC/DataField RubyGems documentation
+  class DataField
+    prepend MARCExtensions::DataFieldExtensions
+  end
+end

data/lib/marc_extensions/field_map.rb

@@ -0,0 +1,63 @@
+require 'marc'
+module MARCExtensions
+  module FieldMapExtensions
+
+    VALID_TAGS = ('000'..'999').freeze
+
+    # Gets the specified fields in order by tag.
+    #
+    # @overload each_sorted_by_tag(tags, &block)
+    #   Yields each specified field.
+    #   @param tags [String, Enumerable<String>] A tag, range of tags, array of tags, or similar
+    #   @yieldparam field [MARC::ControlField, MARC::DataField] Each field.
+    # @overload each_sorted_by_tag(tags)
+    #   An enumerator of the specified variable fields, sorted by tag.
+    #   @param tags [String, Enumerable<String>] A tag, range of tags, array of tags, or similar
+    #   @return [Enumerator::Lazy<MARC::ControlField, MARC::DataField>] the fields
+    # @overload each_sorted_by_tag(&block)
+    #   Yields all fields, sorted by tag.
+    #   @yieldparam field [MARC::ControlField, MARC::DataField] Each field.
+    # @overload each_sorted_by_tag
+    #   An enumerator of all fields, sorted by tag.
+    #   @return [Enumerator::Lazy<MARC::ControlField, MARC::DataField>] the fields
+    def each_sorted_by_tag(tags = nil, &block)
+      reindex unless @clean
+
+      indices_for(tags).map { |i| self[i] }.each(&block)
+    end
+
+    private
+
+    def indices_for(tags)
+      return all_indices unless tags
+
+      sorted_tag_array(tags)
+        .lazy                                      # prevent unnecessary allocations
+        .map { |t| @tags[t] }                      # get indices for each tag
+        .reject(&:nil?)                            # ignoring any tags we don't have fields for
+        .flat_map { |x| x }                        # flatten list of indices -- equiv. Array#flatten
+    end
+
+    def all_indices
+      [].tap do |a|
+        @tags.keys.sort.map do |t|
+          a.concat(@tags[t])
+        end
+      end
+    end
+
+    def sorted_tag_array(tags)
+      return Array(tags) if tags.is_a?(Range)
+
+      Array(tags).sort
+    end
+
+  end
+end
+
+module MARC
+  # @see https://rubydoc.info/gems/marc/MARC/FieldMap RubyGems documentation
+  class FieldMap
+    prepend MARCExtensions::FieldMapExtensions
+  end
+end

data/lib/marc_extensions/record.rb

@@ -0,0 +1,100 @@
+require 'marc'
+require 'marc_extensions/field_map'
+require 'marc_extensions/data_field'
+
+module MARCExtensions
+  module RecordExtensions
+
+    # Gets the specified fields in order by tag.
+    #
+    # @see FieldMapExtensions#each_sorted_by_tag
+    # @overload each_sorted_by_tag(tags, &block)
+    #   Yields each specified field.
+    #   @param tags [String, Enumerable<String>] A tag, range of tags, array of tags, or similar
+    #   @yieldparam field [MARC::ControlField, MARC::DataField] Each field.
+    # @overload each_sorted_by_tag(tags)
+    #   An enumerator of the specified variable fields, sorted by tag.
+    #   @param tags [String, Enumerable<String>] A tag, range of tags, array of tags, or similar
+    #   @return [Enumerator::Lazy<MARC::ControlField, MARC::DataField>] the fields
+    # @overload each_sorted_by_tag(&block)
+    #   Yields all fields, sorted by tag.
+    #   @yieldparam field [MARC::ControlField, MARC::DataField] Each field.
+    # @overload each_sorted_by_tag
+    #   An enumerator of all fields, sorted by tag.
+    #   @return [Enumerator::Lazy<MARC::ControlField, MARC::DataField>] the fields
+    def each_sorted_by_tag(tags = nil, &block)
+      @fields.each_sorted_by_tag(tags, &block)
+    end
+
+    # Gets only the control fields (tag 000-009) from the record. (Note that
+    # this method does not protect against pathological records with data
+    # fields in the control field range.)
+    #
+    # @overload each_control_field
+    #   An enumerator of the control fields.
+    #   @return [Enumerator::Lazy<MARC::ControlField>] the fields
+    # @overload each_control_field(&block)
+    #   Yields each control field.
+    #   @yieldparam field [MARC::ControlField] Each control field.
+    def each_control_field(&block)
+      each_sorted_by_tag.take_while { |df| df.tag.to_i <= 10 }.each(&block)
+    end
+
+    # Gets only the data fields (tag 010-999) from the record. (Note that
+    # this method does not protect against pathological records with control
+    # fields in the data field range.)
+    #
+    # @overload each_data_field
+    #   An enumerator of the data fields.
+    #   @return [Enumerator::Lazy<MARC::DataField>] the fields
+    # @overload each_data_field(&block)
+    #   Yields each data field.
+    #   @yieldparam field [MARC::DataField] Each data field.
+    def each_data_field(&block)
+      each_sorted_by_tag.select { |df| df.tag.to_i > 10 }.each(&block)
+    end
+
+    # Gets the data fields from the record and groups them by tag.
+    #
+    # @return [Hash<String, Array<MARC::DataField>>] a hash from tags to fields
+    def data_fields_by_tag
+      # noinspection RubyYardReturnMatch
+      each_data_field.with_object({}) { |df, t2df| (t2df[df.tag] ||= []) << df }
+    end
+
+    # Gets only the data fields (tag 010-999) from the record. (Note that
+    # this method does not protect against pathological records with control
+    # fields in the data field range.)
+    #
+    # @return [Array<DataField>] the data fields.
+    def data_fields
+      data_fields_by_tag.values.flatten
+    end
+
+    # Freezes the leader and fields.
+    def freeze
+      leader.freeze
+      fields.each(&:freeze)
+      fields.freeze
+      self
+    end
+
+    # @return [Boolean] true if the fields and leader are frozen
+    def frozen?
+      (fields.frozen? && leader.frozen?)
+    end
+
+    # TODO: use info from parsed documentation? or move to TIND-specific extension
+    def record_id
+      cf_001 = self['001']
+      return cf_001.value if cf_001
+    end
+  end
+end
+
+module MARC
+  # @see https://rubydoc.info/gems/marc/MARC/Record RubyGems documentation
+  class Record
+    prepend MARCExtensions::RecordExtensions
+  end
+end

data/lib/marc_extensions/subfield.rb

@@ -0,0 +1,21 @@
+require 'marc'
+
+module MARCExtensions
+  module SubfieldExtensions
+    def frozen?
+      [code, value].all?(&:frozen?)
+    end
+
+    def freeze
+      [code, value].each(&:freeze)
+      self
+    end
+  end
+end
+
+module MARC
+  # @see https://rubydoc.info/gems/marc/MARC/Subfield RubyGems documentation
+  class Subfield
+    prepend MARCExtensions::SubfieldExtensions
+  end
+end

data/lib/marc_extensions/xml_reader.rb

@@ -0,0 +1,19 @@
+require 'marc'
+require 'marc_extensions/record'
+
+module MARCExtensions
+  module XMLReaderClassExtensions
+    def read(file, freeze: false)
+      new(file, freeze: freeze)
+    end
+  end
+end
+
+module MARC
+  # @see https://rubydoc.info/gems/marc/MARC/XMLReader RubyGems documentation
+  class XMLReader
+    class << self
+      prepend MARCExtensions::XMLReaderClassExtensions
+    end
+  end
+end

data/rakelib/bundle.rake

@@ -0,0 +1,8 @@
+namespace :bundle do
+  desc 'Updates the ruby-advisory-db then runs bundle-audit'
+  task :audit do
+    require 'bundler/audit/cli'
+    Bundler::Audit::CLI.start ['update']
+    Bundler::Audit::CLI.start %w[check --ignore CVE-2015-9284]
+  end
+end

data/rakelib/coverage.rake

@@ -0,0 +1,11 @@
+require 'ci/reporter/rake/rspec'
+
+# Configure CI::Reporter report generation
+ENV['GENERATE_REPORTS'] ||= 'true'
+ENV['CI_REPORTS'] = 'artifacts/rspec'
+
+desc 'Run all specs in spec directory, with coverage'
+task coverage: ['ci:setup:rspec'] do
+  ENV['COVERAGE'] ||= 'true'
+  Rake::Task[:spec].invoke
+end

data/rakelib/gem.rake

@@ -0,0 +1,54 @@
+require 'rubygems/gem_runner'
+require 'berkeley_library/marc/module_info'
+
+gem_root_module = BerkeleyLibrary::Marc
+
+class << gem_root_module
+  def project_root
+    @project_root ||= File.expand_path('..', __dir__)
+  end
+
+  def artifacts_dir
+    return project_root unless ENV['CI']
+
+    @artifacts_dir ||= File.join(project_root, 'artifacts')
+  end
+
+  def gemspec_file
+    @gemspec_file ||= begin
+      gemspec_files = Dir.glob(File.expand_path('*.gemspec', project_root))
+      raise ArgumentError, "Too many .gemspecs: #{gemspec_files.join(', ')}" if gemspec_files.size > 1
+      raise ArgumentError, 'No .gemspec file found' if gemspec_files.empty?
+
+      gemspec_files[0]
+    end
+  end
+
+  def gemspec_basename
+    File.basename(gemspec_file)
+  end
+
+  def output_file
+    @output_file ||= begin
+      gem_name = File.basename(gemspec_file, '.*')
+      version = self::ModuleInfo::VERSION
+      basename = "#{gem_name}-#{version}.gem"
+      File.join(artifacts_dir, basename)
+    end
+  end
+
+  def output_file_relative
+    return File.basename(output_file) unless ENV['CI']
+
+    @output_file_relative ||= begin
+      artifacts_dir_relative = File.basename(artifacts_dir)
+      File.join(artifacts_dir_relative, File.basename(output_file))
+    end
+  end
+end
+
+desc "Build #{gem_root_module.gemspec_basename} as #{gem_root_module.output_file_relative}"
+task :gem do
+  args = ['build', gem_root_module.gemspec_file, "--output=#{gem_root_module.output_file}"]
+  Gem::GemRunner.new.run(args)
+end

data/rakelib/rubocop.rake

@@ -0,0 +1,18 @@
+require 'rubocop'
+require 'rubocop/rake_task'
+
+desc 'Run rubocop with HTML output'
+RuboCop::RakeTask.new(:rubocop) do |cop|
+  output = ENV['RUBOCOP_OUTPUT'] || 'artifacts/rubocop/index.html'
+  puts "Writing RuboCop inspection report to #{output}"
+
+  cop.verbose = false
+  cop.formatters = ['html']
+  cop.options = ['--out', output]
+end
+
+desc 'Run RuboCop with auto-correct, and output results to console'
+task :ra do
+  # b/c we want console output, we can't just use `rubocop:auto_correct`
+  RuboCop::CLI.new.run(['--safe-auto-correct'])
+end

data/rakelib/spec.rake

@@ -0,0 +1,2 @@
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)

data/spec/.rubocop.yml

@@ -0,0 +1,37 @@
+inherit_from: ../.rubocop.yml
+
+AllCops:
+  # Exclude generated files
+  Exclude:
+    - 'suite/**/*'
+
+Style/MultilineBlockChain:
+  Enabled: false
+
+Style/ParallelAssignment:
+  Enabled: false
+
+Layout/LineLength:
+  Enabled: false
+
+Metrics/AbcSize:
+  Enabled: false
+
+Metrics/BlockLength:
+  Enabled: false
+
+Metrics/ClassLength:
+  Enabled: false
+
+Metrics/ModuleLength:
+  Enabled: false
+
+Metrics/MethodLength:
+  Enabled: false
+
+############################################################
+# Added in Rubocop 0.89
+
+# Sometimes we're testing the operator
+Lint/BinaryOperatorWithIdenticalOperands:
+  Enabled: false

data/spec/berkeley_library/marc/field_info/var_fields/var_field_def_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+module BerkeleyLibrary
+  module Marc
+    module FieldInfo
+      module VarFields
+        describe VarFieldDef do
+          describe 'to_s' do
+            let(:fields) { %w[046 048].map { |t| [t, VarFields.standard.find { |vf| vf.tag == t }] }.to_h }
+
+            it 'returns something like an EBCDList' do
+              aggregate_failures('to_s') do
+                fields.each do |tag, vf|
+                  expected = File.read("spec/data/field_info/vf_#{tag}.txt").strip
+                  actual = vf.to_s.strip
+                  File.write("tmp/actual_#{tag}.txt", actual) unless actual == expected
+                  expect(actual).to eq(expected)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/spec/berkeley_library/marc/field_info/var_fields/var_field_parser_spec.rb

@@ -0,0 +1,596 @@
+require 'spec_helper'
+require 'parslet/rig/rspec'
+
+require 'berkeley_library/marc/field_info/var_fields/var_field_parser'
+
+module BerkeleyLibrary
+  module Marc
+    module FieldInfo
+      module VarFields
+        describe VarFieldParser do
+          let(:parser) { VarFieldParser.new }
+          let(:printable_chars) { [0x21..0x3f, 0x5b..0x7b, 0x7d..0x7e].map(&:to_a).flatten.map { |cp| cp.chr(Encoding::UTF_8) } }
+
+          describe(:blank) do
+            it 'matches space' do
+              expect(parser.blank).to parse(' ', trace: true)
+            end
+
+            it 'matches tab' do
+              expect(parser.blank).to parse("\t", trace: true)
+            end
+          end
+
+          describe(:printable) do
+            it 'matches space' do
+              expect(parser.printable).to parse(' ', trace: true)
+            end
+
+            it 'matches tab' do
+              expect(parser.printable).to parse("\t", trace: true)
+            end
+
+            it 'matches ASCII printable, except hyphens' do
+              aggregate_failures do
+                nonhyphens = printable_chars.reject { |x| x == '-' }
+                nonhyphens.each { |c| expect(parser.printable).to parse(c, trace: true) }
+              end
+            end
+
+            it 'does not match terminal hyphens' do
+              expect(parser.printable).not_to parse('-', trace: true)
+            end
+          end
+
+          describe(:text) do
+            it 'matches text' do
+              expect(parser.text).to parse('Undefined', trace: true)
+            end
+
+            it 'matches internal hyphens' do
+              txts = ['SERIES ADDED ENTRY--PERSONAL NAME (R)', 'Variable control fields (002-009)']
+              txts.each { |txt| expect(parser.text).to parse(txt, trace: true) }
+            end
+
+            it 'matches leading hyphens' do
+              expect(parser.text).to parse('--Series Added Entry Fields (80X-830)', trace: true)
+            end
+
+            it 'does not match terminal hyphens' do
+              expect(parser.text).not_to parse('Series Added Entry Fields (80X-830)--', trace: true)
+            end
+          end
+
+          describe(:ln_br) do
+            it 'matches a Windows line break' do
+              expect(parser.ln_br).to parse("\r\n", trace: true)
+            end
+
+            it 'matches Unicode newline characters' do
+              chars = [0xa, 0xb, 0xc, 0xd, 0x85, 0x2028, 0x2029].map { |cp| cp.chr(Encoding::UTF_8) }
+              aggregate_failures do
+                chars.each { |c| expect(parser.ln_br).to parse(c, trace: true) }
+              end
+            end
+          end
+
+          describe(:eol) do
+            it 'matches a newline' do
+              expect(parser.eol).to parse("\n", trace: true)
+            end
+
+            it 'matches a newline with leading whitespace' do
+              expect(parser.eol).to parse("  \t\n", trace: true)
+            end
+          end
+
+          describe(:text) do
+            it 'matches an ASCII printable sequence' do
+              expect(parser.text).to parse(printable_chars.join, trace: true)
+            end
+          end
+
+          describe(:comment) do
+            it 'matches a comment' do
+              expect(parser.comment).to parse('// this is a comment', trace: true)
+            end
+
+            it 'captures leading whitespace' do
+              expect(parser.comment).to parse(" \t// this is a comment", trace: true)
+            end
+          end
+
+          describe(:nc) do
+            it 'matches a comment' do
+              expect(parser.nc).to parse('// this is a comment', trace: true)
+            end
+
+            it 'matches a newline' do
+              expect(parser.nc).to parse("\n", trace: true)
+            end
+
+            it 'matches a block of comments and whitespace' do
+              block = <<~TXT
+
+                // this is a comment
+
+                // here is another comment
+                // and another
+
+              TXT
+
+              expect(parser.nc).to parse(block, trace: true)
+            end
+
+            it 'matches the empty string' do
+              expect(parser.nc).to parse('', trace: true)
+            end
+
+            it 'matches indented comments' do
+              block = <<~TXT
+                // this is a comment
+
+                      // this is another comment
+                  // so is this
+              TXT
+
+              expect(parser.nc).to parse(block, trace: true)
+            end
+          end
+
+          describe(:ind_def) do
+            it 'parses an indicator definition' do
+              ind_def = '0 - No added entry'
+              expect(parser.ind_def).to parse(ind_def, trace: true)
+            end
+
+            it 'parses an undefined indicator definition' do
+              ind_def = '# - Undefined'
+              expect(parser.ind_def).to parse(ind_def, trace: true)
+            end
+
+            it 'parses an indicator value range' do
+              ind_def = '0-9 - Number of nonfiling characters present'
+              expect(parser.ind_def).to parse(ind_def, trace: true)
+            end
+          end
+
+          describe(:indicators) do
+            it 'matches an empty indicator definition' do
+              ind_def = <<~TXT.strip
+                Indicators
+                   First - Undefined
+                      # - Undefined
+                   Second - Undefined
+                      # - Undefined
+              TXT
+
+              expect(parser.indicators).to parse(ind_def, trace: true)
+            end
+
+            it 'parses typical indicators' do
+              ind_def = <<~TXT.strip
+                Indicators
+                   First - Access method
+                      # - No information provided
+                      0 - Email
+                      1 - FTP
+                      2 - Remote login (Telnet)
+                      3 - Dial-up
+                      4 - HTTP
+                      7 - Method specified in subfield $2
+                   Second - Relationship
+                      # - No information provided
+                      0 - Resource
+                      1 - Version of resource
+                      2 - Related resource
+                      8 - No display constant generated
+              TXT
+
+              expect(parser.indicators).to parse(ind_def, trace: true)
+            end
+
+            it 'parses obsolete indicators' do
+              ind_def = <<~TXT.strip
+                Indicators
+                   First - Government jurisdiction (BK MP MU VM)[OBSOLETE]
+                      0 - United States [OBSOLETE]
+                      1 - Canada [OBSOLETE] [CAN/MARC only]
+                      2 - France [OBSOLETE] [CAN/MARC only]
+                   First - Undefined
+                      # - Undefined
+                   Second - Undefined [OBSOLETE]
+                      # - Undefined [OBSOLETE]
+                   Second - Display constant controller
+                      # - Copyright or legal deposit number
+                      8 - No display constant controller generated
+              TXT
+
+              expect(parser.indicators).to parse(ind_def, trace: true)
+            end
+
+            it 'parses obsolete indicators with no values' do
+              ind_def = <<~TXT.strip
+                Indicators
+                   First - Number source
+                      # - Source specified in subfield $2
+                      # - Undefined (BK MP MU VM SE) [OBSOLETE]
+                      0 - Superintendent of Documents Classification System
+                      1 - Government of Canada Publications: Outline of Classification
+                   First - Government jurisdiction (BK MP MU VM SE) [OBSOLETE]
+                   Second - Undefined
+                      # - Undefined
+              TXT
+
+              expect(parser.indicators).to parse(ind_def, trace: true)
+            end
+
+            it 'parses value ranges' do
+              ind_def = <<~TXT.strip
+                Indicators
+                   First - Nonfiling characters
+                      0-9 - Number of nonfiling characters present
+                      # - Nonfiling characters not specified [OBSOLETE]
+                   Second - Undefined
+                      # - Undefined
+                   Second - Main entry/subject relationship (BK MU SE) [OBSOLETE]
+              TXT
+
+              expect(parser.indicators).to parse(ind_def, trace: true)
+            end
+          end
+
+          describe(:subfield_value) do
+            it 'parses a single-character value' do
+              expect(parser.subfield_value).to parse('1 - Form of name', trace: true)
+            end
+
+            it 'parses a multi-character value' do
+              expect(parser.subfield_value).to parse('isds/c - ISSN Canada', trace: true)
+            end
+          end
+
+          describe(:subfield_def) do
+            it 'parses a subfield code definition without values' do
+              subfield_def = '$4 - Relationship (R)'
+              expect(parser.subfield_def).to parse(subfield_def, trace: true)
+            end
+
+            it 'parses a subfield code definition with values' do
+              subfield_def = <<~TXT.strip
+                $7 - Control subfield (NR)
+                   0 - Type of main entry heading
+                   1 - Form of name
+              TXT
+
+              expect(parser.subfield_def).to parse(subfield_def, trace: true)
+            end
+
+            it "doesn't consume next vf definition" do
+              non_def = <<~TXT.strip
+                      $8 - Field link and sequence number (R)
+
+                011 - LINKING LIBRARY OF CONGRESS CONTROL NUMBER (NR) [OBSOLETE]
+              TXT
+              expect(parser.subfield_def).not_to parse(non_def, trace: true)
+            end
+          end
+
+          describe(:subfield_codes) do
+            it 'parses a typical set of subfield codes' do
+              subfield_codes = <<~TXT.strip
+                Subfield Codes
+                   $a - Replacement title (R)
+                   $i - Explanatory text (R)
+                   $w - Replacement bibliographic record control number (R)
+                   $6 - Linkage (NR)
+                   $8 - Field link and sequence number (R)
+              TXT
+
+              expect(parser.subfield_codes).to parse(subfield_codes, trace: true)
+            end
+
+            it 'parses a range of subfield codes' do
+              subfield_codes = <<~TXT.strip
+                Subfield Codes
+                   $a - Tag of the foreign MARC field (NR)
+                   $b - Content of the foreign MARC field (NR)
+                   $2 - Source of data (NR)
+                   $a-z - Foreign MARC subfield (R)
+                   $0-9 - Foreign MARC subfield (R)
+              TXT
+
+              expect(parser.subfield_codes).to parse(subfield_codes, trace: true)
+            end
+
+            it 'parses a range of subfield codes with missing hyphen before desc' do
+              subfield_codes = <<~TXT.strip
+                Subfield Codes
+                   $6 - Linkage (NR)
+                   $a-z Same as associated field
+                   $0-5 Same as associated field
+              TXT
+
+              expect(parser.subfield_codes).to parse(subfield_codes, trace: true)
+            end
+          end
+
+          describe(:ivc_def) do
+            it 'parses instrument or voices codes' do
+              ivc_def = <<~TXT.strip
+                Instrument or Voices Codes
+                   ba - Brass - Horn
+                   bb - Brass--Trumpet
+                   bc - Brass--Cornet
+              TXT
+
+              expect(parser.ivc_def).to parse(ivc_def, trace: true)
+            end
+          end
+
+          describe(:vf) do
+            it 'parses a typical field' do
+              vf = <<~TXT.strip
+                886 - FOREIGN MARC INFORMATION FIELD (R)
+                   Indicators
+                      First - Type of field
+                         0 - Leader
+                         1 - Variable control fields (002-009)
+                         2 - Variable data fields (010-999)
+                      Second - Undefined
+                         # - Undefined
+                   Subfield Codes
+                      $a - Tag of the foreign MARC field (NR)
+                      $b - Content of the foreign MARC field (NR)
+                      $2 - Source of data (NR)
+                      $a-z - Foreign MARC subfield (R)
+                      $0-9 - Foreign MARC subfield (R)
+              TXT
+
+              expect(parser.vf).to parse(vf, trace: true)
+            end
+
+            it 'parses a field with no indicators or subfields' do
+              vf = '863 - ENUMERATION AND CHRONOLOGY--BASIC BIBLIOGRAPHIC UNIT (R)'
+
+              expect(parser.vf).to parse(vf, trace: true)
+            end
+
+            it 'parsers a fields with Instrument or Voices Codes' do
+              vf = <<~TXT.strip
+                048 - NUMBER OF MUSICAL INSTRUMENTS OR VOICES CODE (R)
+                   Indicators
+                      First - Undefined
+                         # - Undefined
+                      Second - Source of code
+                         # - MARC code
+                         7 - Source specified in subfield $2
+                   Subfield Codes
+                      $a - Performer or ensemble (R)
+                      $b - Soloist (R)
+                      $2 - Source of code (NR)
+                      $8 - Field link and sequence number (R)
+                   Instrument or Voices Codes
+                      ba - Brass - Horn
+                      bb - Brass--Trumpet
+              TXT
+
+              expect(parser.vf).to parse(vf, trace: true)
+            end
+
+            it 'parses a TIND field with bogus indicators' do
+              vf = <<~TXT.strip
+                909 - OAI REPOSITORY METADATA (NR)
+                  Indicators
+                     First - ???
+                        C - ??? // Yes, this should be invalid; yes, TIND does it anyway
+                     Second - ???
+                        0 - ???
+                  Subfield Codes
+                     $o - OAI URI for the record (oai:<hostname>:<record ID>) (NR)
+                     $p - setSpec for current valid sets (NR)
+                     $q - setSpec for previously valid sets (NR)
+              TXT
+
+              expect(parser.vf).to parse(vf, trace: true)
+            end
+          end
+
+          describe(:section_header) do
+            it 'matches a section header' do
+              expect(parser.section_header).to parse('--Number and Code Fields (01X-04X)--', trace: true)
+            end
+          end
+
+          describe(:section) do
+            it 'matches a section with header' do
+              section = <<~TXT.strip
+                --Number and Code Fields (01X-04X)--
+                010 - LIBRARY OF CONGRESS CONTROL NUMBER (NR)
+                   Indicators
+                      First - Undefined
+                         # - Undefined
+                      Second - Undefined
+                         # - Undefined
+                   Subfield Codes
+                      $a - LC control number (NR)
+                      $b - NUCMC control number (R)
+                      $z - Canceled/invalid LC control number (R)
+                      $8 - Field link and sequence number (R)
+
+                011 - LINKING LIBRARY OF CONGRESS CONTROL NUMBER (NR) [OBSOLETE]
+                   Indicators
+                      First - Undefined
+                         # - Undefined
+                      Second - Undefined
+                         # - Undefined
+                   Subfield Codes
+                      $a - LINKING LC control number (R)
+              TXT
+
+              expect(parser.section).to parse(section, trace: true)
+            end
+
+            it 'matches a section without header' do
+              section = <<~TXT.strip
+                852 - LOCATION (R)
+                   Indicators
+                      First - Undefined
+                         # - Undefined
+                      Second - Undefined
+                         # - Undefined
+                   Subfield Codes
+                      $c - Location/Archive/Repository
+
+                901 - IDENTIFIERS [INTERNAL] (NR) [REQUIRED WHEN APPLICABLE]
+                   Indicators
+                      First - Undefined
+                         # - Undefined
+                      Second - Undefined
+                         # - Undefined
+                   Subfield Codes
+                      $a - Ark identifier for METS file [REQUIRED WHEN APPLICABLE]
+                      $f - Ark identifier for finding aid
+                      $g - PJID:DBID from GenDB [REQUIRED WHEN APPLICABLE]
+                      $m - Millennium record number
+                      $o - OCLC Number
+              TXT
+
+              expect(parser.section).to parse(section, trace: true)
+            end
+          end
+
+          describe(:list) do
+            it 'parses multiple sections' do
+              sections = <<~TXT.strip
+                --Number and Code Fields (01X-04X)--
+                010 - LIBRARY OF CONGRESS CONTROL NUMBER (NR)
+                   Indicators
+                      First - Undefined
+                         # - Undefined
+                      Second - Undefined
+                         # - Undefined
+                   Subfield Codes
+                      $a - LC control number (NR)
+                      $8 - Field link and sequence number (R)
+
+                048 - NUMBER OF MUSICAL INSTRUMENTS OR VOICES CODE (R)
+                   Indicators
+                      First - Undefined
+                         # - Undefined
+                      Second - Source of code
+                         7 - Source specified in subfield $2
+                   Subfield Codes
+                      $a - Performer or ensemble (R)
+                      $8 - Field link and sequence number (R)
+                   Instrument or Voices Codes
+                      bc - Brass--Cornet
+                      bd - Brass--Trombone
+
+
+                --Classification and Call Number Fields (05X-08X)--
+                050 - LIBRARY OF CONGRESS CALL NUMBER (R)
+                   Indicators
+                      First - Existence in LC collection
+                         0 - Item is in LC
+                         1 - Item is not in LC
+                      Second - Source of call number
+                         0 - Assigned by LC
+                         4 - Assigned by agency other than LC
+                      Second - Series call number (SE) [OBSOLETE]
+                         0 - No series involved
+                         1 - Main series
+                   Subfield Codes
+                      $a - Classification number (R)
+                      $b - Item number (NR)
+              TXT
+
+              expect(parser.list).to parse(sections, trace: true)
+            end
+
+          end
+
+          describe(:parse) do
+            it 'parses the standard list' do
+              list = File.read(VarFields::PATH_STANDARD)
+              expect(parser).to parse(list, trace: true)
+
+              parse_tree = parser.parse(list)
+              expect(parse_tree.keys).to contain_exactly(:doc_comments, :sections)
+
+              doc_comments = parse_tree[:doc_comments]
+              expected_comments = [
+                'MARC 21 Format for Bibliographic Data Field List',
+                'Adapted from https://www.loc.gov/marc/bibliographic/ecbdlist.html',
+                'Retrieved 2021-01-13'
+              ]
+
+              expect(doc_comments.size).to eq(3)
+              expect(doc_comments.map { |c| c[:comment] }).to eq(expected_comments)
+
+              sections = parse_tree[:sections]
+              expected_sections = [
+                'Number and Code Fields (01X-04X)',
+                'Classification and Call Number Fields (05X-08X)',
+                'Main Entry Fields (1XX)',
+                'Title and Title-Related Fields (20X-24X)',
+                'Edition, Imprint, etc. Fields 250-270',
+                'Physical Description, etc. Fields (3XX)',
+                'Series Statement Fields (4XX)',
+                'Note Fields (Part 1: 50X-53X)',
+                'Note Fields (Part 2: 53X-58X)',
+                '59X - LOCAL NOTES',
+                'Subject Access Fields (6XX)',
+                'Added Entry Fields (70X-75X)',
+                'Linking Entry Fields (76X-78X)',
+                'Series Added Entry Fields (80X-830)',
+                'Holdings, Location, Alternate Graphics, etc. Fields (841-88X)'
+              ]
+              expect(sections.map { |s| s[:desc] }).to eq(expected_sections)
+            end
+
+            describe 'non-standard lists' do
+              data_dir = File.dirname(VarFields::PATH_STANDARD)
+              Dir.glob(File.expand_path('var_fields_*.txt', data_dir)).each do |p|
+                next if p == VarFields::PATH_STANDARD
+
+                basename = File.basename(p)
+                it "parses #{basename}" do
+                  list = File.read(p)
+                  expect(parser).to parse(list, trace: true)
+
+                  # parse_tree = parser.parse(list)
+                  # const_name = basename.sub(/^var_fields_/, '').sub(/\.txt/, '_parsed').upcase
+                  # header = <<~HEADER
+                  #     module UCBLIT
+                  #       module Marc
+                  #         module FieldInfo
+                  #           module VarFields
+                  #             #{const_name} =
+                  # HEADER
+                  # footer = <<~FOOTER
+                  #           end
+                  #         end
+                  #       end
+                  #     end
+                  # FOOTER
+                  #
+                  # parse_tree_src = StringIO.new.tap do |out|
+                  #   out.puts(header)
+                  #   PP.pp(parse_tree, out)
+                  #   out.puts(footer)
+                  # end.string.gsub(/@[0-9]+/, '')
+                  #
+                  # parsed_file = p.sub(/\.txt/, '_parsed.rb')
+                  # File.write(parsed_file, parse_tree_src)
+                end
+              end
+            end
+
+          end
+
+        end
+      end
+    end
+  end
+end