berkeley_library-tind 0.4.0

data/lib/berkeley_library/tind/export/csv_exporter.rb

@@ -0,0 +1,29 @@
+require 'csv'
+require 'berkeley_library/tind/export/exporter'
+
+module BerkeleyLibrary
+  module TIND
+    module Export
+      # Exporter for CSV (comma-separated value) text
+      class CSVExporter < Exporter
+        # Exports {ExportBase#collection} as CSV
+        # @overload export
+        #   Exports to a new string.
+        #   @return [String] the CSV string
+        # @overload export(out)
+        #   Exports to the specified output stream.
+        #   @param out [IO] the output stream
+        #   @return[void]
+        # @overload export(path)
+        #   Exports to the specified file.
+        #   @param path [String, Pathname] the path to the output file
+        #   @return[void]
+        def export(out = nil)
+          # noinspection RubyYardReturnMatch
+          export_table.tap { logger.info('Writing CSV') }.to_csv(out)
+        end
+
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/export/export.rb

@@ -0,0 +1,47 @@
+require 'berkeley_library/tind/export/export_format'
+
+module BerkeleyLibrary
+  module TIND
+    module Export
+      class << self
+        # Writes a spreadsheet in the specified format
+        # @overload export(collection, format = ExportFormat::CSV)
+        #   Returns the spreadsheet as a string.
+        #   @param collection [String] The collection name
+        #   @param format [ExportFormat, String, Symbol] the export format
+        #   @param exportable_only [Boolean] whether to include only exportable fields
+        # @overload export(collection, format = ExportFormat::CSV, out)
+        #   Writes the spreadsheet to the specified output stream.
+        #   @param collection [String] The collection name
+        #   @param format [ExportFormat, String, Symbol] the export format
+        #   @param out [IO] the output stream
+        #   @param exportable_only [Boolean] whether to include only exportable fields
+        # @overload export(collection, format = ExportFormat::CSV, path)
+        #   Writes the spreadsheet to the specified output file.
+        #   @param collection [String] The collection name
+        #   @param format [ExportFormat, String, Symbol] the export format
+        #   @param path [String, Pathname] the path to the output file
+        #   @param exportable_only [Boolean] whether to include only exportable fields
+        # @raise [ExportFailed] if the collection does not exist, or cannot be exported. Note
+        #   that this error is guaranteed to be raised before anything is written to `out`.
+        def export(collection, format = ExportFormat::CSV, out = nil, exportable_only: true)
+          # noinspection RubyYardParamTypeMatch
+          exporter = exporter_for(collection, format, exportable_only: exportable_only)
+          exporter.export(out)
+        end
+
+        # Returns an exporter for the specified spreadsheet in the specified format
+        # @param collection [String] The collection name
+        # @param format [ExportFormat, String, Symbol] the export format
+        # @param exportable_only [Boolean] whether to include only exportable fields
+        # @return [Exporter] the exporter
+        def exporter_for(collection, format, exportable_only: true)
+          export_format = ExportFormat.ensure_format(format)
+          # noinspection RubyNilAnalysis
+          export_format.exporter_for(collection, exportable_only: exportable_only)
+        end
+
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/export/export_command.rb

@@ -0,0 +1,168 @@
+require 'optparse'
+
+require 'berkeley_library/tind/api'
+require 'berkeley_library/tind/config'
+require 'berkeley_library/tind/export/export'
+require 'berkeley_library/tind/export/export_format'
+require 'berkeley_library/logging'
+require 'berkeley_library/util/sys_exits'
+
+module BerkeleyLibrary
+  module TIND
+    module Export
+      # rubocop:disable Metrics/ClassLength
+      class ExportCommand
+        include BerkeleyLibrary::Util::SysExits
+
+        attr_reader :options
+        attr_reader :out
+
+        def initialize(*args, out: $stdout)
+          @options = ExportCommand.parse_options(args)
+          @out = out
+        end
+
+        def execute!
+          return list_collections if options[:list]
+
+          export_collection
+        rescue StandardError => e
+          warn(e)
+          warn(e.backtrace.join("\n")) if e.backtrace && options[:verbose]
+
+          exit(EX_SOFTWARE)
+        end
+
+        private
+
+        def list_collections
+          BerkeleyLibrary::TIND::API::Collection.each_collection { |c| out.puts "#{c.nb_rec}\t#{c.name}" }
+        end
+
+        def export_collection
+          BerkeleyLibrary::TIND::Export.export(
+            options[:collection],
+            options[:format],
+            options[:outfile] || out
+          )
+        end
+
+        class << self
+          include BerkeleyLibrary::Util::SysExits
+
+          DEFAULT_FORMAT = ExportFormat::CSV
+          FORMATS = ExportFormat.to_a.map(&:value).join(', ')
+          OPTS = {
+            f: ['--format FORMAT', "Format (#{FORMATS}; defaults to output file extension, or else to #{DEFAULT_FORMAT})"],
+            o: ['--output-file FILE', 'Output file or directory'],
+            l: ['--list-collections', 'List collection sizes and names'],
+            u: ['--tind-base-url URL', "TIND base URL (default $#{BerkeleyLibrary::TIND::Config::ENV_TIND_BASE_URL})"],
+            k: ['--api-key KEY', "TIND API key (default $#{BerkeleyLibrary::TIND::Config::ENV_TIND_API_KEY})"],
+            e: ['--env-file [ENV]', 'Read environment variables from <ENV> (default: ./.env)'],
+            v: ['--verbose', 'Verbose error logging'],
+            h: ['--help', 'Show help and exit']
+          }.freeze
+
+          def parse_options(argv)
+            {}.tap do |opts|
+              option_parser(opts).parse!(argv)
+              opts[:collection] = argv.pop
+              opts[:format] = ensure_format(opts)
+              validate!(opts)
+              configure!(opts)
+            end
+          rescue StandardError => e
+            print_usage_and_exit!($stderr, EX_USAGE, e.message)
+          end
+
+          private
+
+          def validate!(opts)
+            return if opts[:list]
+            raise ArgumentError, 'Collection not specified' unless opts[:collection]
+            raise ArgumentError, 'OpenOffice/LibreOffice export requires a filename' if opts[:format] == ExportFormat::ODS && !opts[:outfile]
+          end
+
+          def configure!(opts)
+            configure_env(opts)
+            BerkeleyLibrary::TIND::Config.base_uri = opts[:tind_base_url] if opts[:tind_base_url]
+            BerkeleyLibrary::TIND::Config.api_key = opts[:api_key] if opts[:api_key]
+            BerkeleyLibrary::Logging.logger = configure_logger(opts)
+          end
+
+          def configure_logger(opts)
+            return Logger.new(File::NULL) unless opts[:verbose]
+
+            # TODO: simpler log format? different log levels?
+            BerkeleyLibrary::Logging::Loggers.new_readable_logger($stderr).tap { |logger| logger.level = Logger::DEBUG }
+          end
+
+          def configure_env(opts)
+            return unless (env_file = opts[:env_file])
+
+            warn "Reading environment from #{env_file}" if opts[:verbose]
+
+            require 'dotenv'
+            Dotenv.load(env_file)
+          end
+
+          def ensure_format(opts)
+            fmt = opts[:format] || (File.extname(opts[:outfile]).sub(/^\./, '') if opts[:outfile])
+            return DEFAULT_FORMAT unless fmt
+
+            ExportFormat.ensure_format(fmt)
+          end
+
+          # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+          def option_parser(opts = {})
+            OptionParser.new do |p|
+              p.summary_indent = ' '
+              p.on('-f', *OPTS[:f]) { |fmt| opts[:format] = fmt }
+              p.on('-o', *OPTS[:o]) { |out| opts[:outfile] = out }
+              p.on('-l', *OPTS[:l]) { opts[:list] = true }
+              p.on('-u', *OPTS[:u]) { |url| opts[:tind_base_url] = url }
+              p.on('-k', *OPTS[:k]) { |k| opts[:api_key] = k }
+              p.on('-e', *OPTS[:e]) { |e| opts[:env_file] = env_file_path(e) }
+              p.on('-v', *OPTS[:v]) { opts[:verbose] = true }
+              p.on('-h', *OPTS[:h]) { print_usage_and_exit! }
+            end
+          end
+          # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+
+          def env_file_path(env_file_opt)
+            File.realpath(env_file_opt || File.join(Dir.pwd, '.env'))
+          end
+
+          def print_usage_and_exit!(out = $stdout, exit_code = EX_OK, msg = nil)
+            out.puts("#{msg}\n\n") if msg
+            out.puts(usage)
+            exit(exit_code)
+          end
+
+          def usage
+            <<~USAGE
+              Usage: tind-export [options] COLLECTION
+
+              Options:
+                #{summarize_options}
+
+              Examples:
+                1. list collections
+                   tind-export --list-collections
+                2. export a collection as an OpenOffice/LibreOffice spreadsheet
+                   tind-export -o lincoln-papers.ods 'Abraham Lincoln Papers'
+                3. export a collection as an OpenOffice/LibreOffice spreadsheet in exploded XML format,
+                   where `lincoln-papers` is a directory
+                   tind-export -v -f ODS -o lincoln-papers 'Abraham Lincoln Papers'
+            USAGE
+          end
+
+          def summarize_options
+            option_parser.summarize.join('  ')
+          end
+        end
+      end
+      # rubocop:enable Metrics/ClassLength
+    end
+  end
+end

data/lib/berkeley_library/tind/export/export_exception.rb

@@ -0,0 +1,8 @@
+module BerkeleyLibrary
+  module TIND
+    module Export
+      # Wrapper for export-specific exceptions.
+      class ExportException < StandardError; end
+    end
+  end
+end

data/lib/berkeley_library/tind/export/export_format.rb

@@ -0,0 +1,67 @@
+require 'typesafe_enum'
+require 'berkeley_library/tind/export/csv_exporter'
+require 'berkeley_library/tind/export/ods_exporter'
+
+module BerkeleyLibrary
+  module TIND
+    module Export
+      class ExportFormat < TypesafeEnum::Base
+        new :CSV
+        new :ODS
+
+        DEFAULT = ODS
+
+        def exporter_for(collection, exportable_only: true)
+          return CSVExporter.new(collection, exportable_only: exportable_only) if self == ExportFormat::CSV
+          return ODSExporter.new(collection, exportable_only: exportable_only) if self == ExportFormat::ODS
+        end
+
+        def description
+          return 'CSV (comma-separated text)' if self == ExportFormat::CSV
+          return 'LibreOffice/OpenOffice spreadsheet' if self == ExportFormat::ODS
+        end
+
+        def mime_type
+          return 'text/csv' if self == ExportFormat::CSV
+          return 'application/vnd.oasis.opendocument.spreadsheet' if self == ExportFormat::ODS
+        end
+
+        def to_s
+          # noinspection RubyYardReturnMatch
+          value
+        end
+
+        def to_str
+          value
+        end
+
+        def inspect
+          "#{ExportFormat}::#{key}"
+        end
+
+        def default?
+          self == DEFAULT
+        end
+
+        # noinspection RubyYardReturnMatch
+        class << self
+          # Converts a string or symbol to an {ExportFormat}, or returns
+          # an {ExportFormat} if passed on
+          #
+          # @param format [String, Symbol, ExportFormat] the format
+          # @return [ExportFormat] the format
+          def ensure_format(format)
+            return unless format
+            return format if format.is_a?(ExportFormat)
+
+            fmt = ExportFormat.find_by_value(format.to_s.downcase)
+            return fmt if fmt
+
+            raise ArgumentError, "Unknown #{ExportFormat}: #{format.inspect}"
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/berkeley_library/tind/export/exporter.rb

@@ -0,0 +1,105 @@
+require 'berkeley_library/util/logging'
+require 'berkeley_library/tind/api/search'
+require 'berkeley_library/tind/export/table'
+
+module BerkeleyLibrary
+  module TIND
+    module Export
+
+      # Superclass of exporters for different formats
+      class Exporter
+        include BerkeleyLibrary::Logging
+
+        # ------------------------------------------------------------
+        # Accessors
+
+        # @return [String] the collection name
+        attr_reader :collection
+
+        # @return [Boolean] whether to include only exportable fields
+        attr_reader :exportable_only
+
+        # ------------------------------------------------------------
+        # Initializer
+
+        # Initializes a new exporter
+        #
+        # @param collection [String] The collection name
+        # @param exportable_only [Boolean] whether to include only exportable fields
+        def initialize(collection, exportable_only: true)
+          @collection = collection
+          @exportable_only = exportable_only
+        end
+
+        # ------------------------------------------------------------
+        # Abstract methods
+
+        # Exports the collection
+        # @param out [IO, String, Pathname, nil] the IO or file path to write the
+        #   exported data to, or nil to return a string
+        # rubocop:disable Lint/UnusedMethodArgument
+        def export(out = nil)
+          # This is a stub, used for documentation
+          raise NoMethodError, "#{self.class} does not implement `export`"
+        end
+        # rubocop:enable Lint/UnusedMethodArgument
+
+        # ------------------------------------------------------------
+        # Accessors
+
+        # Returns true if the collection can be exported, false otherwise.
+        # Note that this requires reading the collection data from the TIND
+        # server; failures will be fast but success may be slow. (On the other
+        # hand, the retrieved collection data is cached, so the subsequent
+        # export will not need to retrieve it again.)
+        def any_results?
+          !_export_table.empty?
+        end
+
+        # ------------------------------------------------------------
+        # Object overrides
+
+        def respond_to?(*args)
+          return false if instance_of?(Exporter) && (args && args.first.to_s == 'export')
+
+          super
+        end
+
+        # ------------------------------------------------------------
+        # Protected methods
+
+        protected
+
+        # Returns a table of all records in the specified
+        # collection
+        #
+        # @return [Export::Table] the table
+        # @raise NoResultsError if no search results were returned for the collection
+        def export_table
+          # TODO: something more clever. Search.has_results?
+          return _export_table unless _export_table.empty?
+
+          raise no_results_error
+        end
+
+        private
+
+        def no_results_error
+          NoResultsError.new("No records returned for collection #{collection.inspect}")
+        end
+
+        def _export_table
+          @_export_table ||= begin
+            logger.info("Reading collection #{collection.inspect}")
+            results = API::Search.new(collection: collection).each_result(freeze: true)
+
+            logger.info('Creating export table')
+            # noinspection RubyYardParamTypeMatch
+            Export::Table.from_records(results, freeze: true, exportable_only: exportable_only)
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/berkeley_library/tind/export/filter.rb

@@ -0,0 +1,52 @@
+module BerkeleyLibrary
+  module TIND
+    module Export
+      module Filter
+        DO_NOT_EXPORT_FIELDS = ['005', '8564 ', '902  ', '903  ', '991', '998'].map(&:freeze).freeze
+        DO_NOT_EDIT_FIELDS = (['001'.freeze] + DO_NOT_EXPORT_FIELDS).freeze
+
+        DO_NOT_EXPORT_SUBFIELDS = ['336  a', '852  c', '901  a', '901  f', '901  g', '980  a', '982  a', '982  b', '982  p'].map(&:freeze).freeze
+        DO_NOT_EDIT_SUBFIELDS = (['035  a'.freeze] + DO_NOT_EXPORT_SUBFIELDS).freeze
+
+        DO_NOT_EDIT = (DO_NOT_EDIT_FIELDS + DO_NOT_EDIT_SUBFIELDS).freeze
+
+        class << self
+          def can_export_tag?(tag)
+            !DO_NOT_EXPORT_FIELDS.include?(tag)
+          end
+
+          def can_export_data_field?(df)
+            !exportable_subfield_codes(df).empty?
+          end
+
+          def exportable_subfield_codes(df)
+            tag, ind1, ind2 = decompose_data_field(df)
+            DO_NOT_EXPORT_FIELDS.each { |f| return [] if excludes?(f, tag, ind1, ind2) }
+
+            df.subfield_codes.reject do |code|
+              DO_NOT_EXPORT_SUBFIELDS.any? { |f| excludes?(f, tag, ind1, ind2, code) }
+            end
+          end
+
+          def can_edit?(tag, ind1, ind2, code)
+            DO_NOT_EDIT.none? { |f| excludes?(f, tag, ind1, ind2, code) }
+          end
+
+          private
+
+          def decompose_data_field(df)
+            [df.tag, df.indicator1, df.indicator2]
+          end
+
+          # TODO: test this more carefully
+          def excludes?(f, tag, ind1, ind2, code = nil)
+            return f == tag if f.size == 3
+
+            excludes_tag = f.start_with?(tag) && f[3] == ind1 && f[4] == ind2
+            code ? excludes_tag && code : excludes_tag
+          end
+        end
+      end
+    end
+  end
+end