overdrive_metadata 1.0.0

data/.autotest

@@ -0,0 +1,23 @@
+# -*- ruby -*-
+
+require 'autotest/restart'
+
+# Autotest.add_hook :initialize do |at|
+#   at.extra_files << "../some/external/dependency.rb"
+#
+#   at.libs << ":../some/external"
+#
+#   at.add_exception 'vendor'
+#
+#   at.add_mapping(/dependency.rb/) do |f, _|
+#     at.files_matching(/test_.*rb$/)
+#   end
+#
+#   %w(TestA TestB).each do |klass|
+#     at.extra_class_map[klass] = "test/test_misc.rb"
+#   end
+# end
+
+# Autotest.add_hook :run_command do |at|
+#   system "rake build"
+# end

data/History.txt

@@ -0,0 +1,2 @@
+=== 1.0.0 / 2011-10-27
+

data/Manifest.txt

@@ -0,0 +1,8 @@
+.autotest
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+lib/overdrive_metadata.rb
+raw/test.xls
+test/test_overdrive_metadata.rb

data/README.txt

@@ -0,0 +1,62 @@
+= overdrive_metadata
+
+http://www.libcode.net
+
+== DESCRIPTION:
+
+Generate marc records from Overdrive provided metadata spreadsheets.
+
+== FEATURES/PROBLEMS:
+
+Have yet to see a Kindle eBook sample - may require tinkering.
+
+== SYNOPSIS:
+
+require 'overdrive_metadata'
+records = OverdriveMetadata.new('spreadsheets/111111.xls')
+puts "R: " + records.size.to_s # print number of records generated to console
+w = MARC::Writer.new('generated.mrc')
+records.each do |r|
+  begin
+    w.write r
+  rescue
+    puts "FAILED: " + r['245']['a']
+  end
+end
+w.close
+
+== REQUIREMENTS:
+
+htmlentities
+marc
+sanitize
+spreadsheet
+
+== INSTALL:
+
+sudo gem install overdrive_metadata
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2011 Mark Cooper
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,13 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+
+Hoe.plugin :gemspec
+Hoe.plugin :git
+
+Hoe.spec 'overdrive_metadata' do
+	developer('Mark Cooper', 'markchristophercooper@gmail.com')
+end
+
+# vim: syntax=ruby

data/lib/overdrive_metadata.rb

@@ -0,0 +1,334 @@
+require 'htmlentities'
+require 'marc'
+require 'sanitize'
+require 'spreadsheet'
+
+##
+# Class to generate marc records from Overdrive provided metadata spreadsheet
+# Usage:
+# require 'overdrive_metadata'
+# records = OverdriveMetadata.new('spreadsheets/111111.xls')
+# puts "R: " + records.size.to_s # print number of records generated to console
+# w = MARC::Writer.new('generated.mrc')
+# records.each do |r|
+#   begin
+#     w.write r
+#   rescue
+#     puts "FAILED: " + r['245']['a']
+#   end
+# end
+# w.close
+
+class OverdriveMetadata
+    VERSION = '1.0.0'
+
+    attr_reader :records
+
+    DEF_FORMAT = 'eBook'
+    OD_ORG     = 'OverDrive, Inc.'
+    OD_URL     = 'http://www.overdrive.com'
+    AGENCY     = 'JTH' # Sonoma County Library
+    ACCESS     = 'Mode of access: World Wide Web.'
+    URL_MSG    = 'Click to download this resource.'
+    DISCLAIM   = 'Record generated from Overdrive metadata spreadsheet.'
+    READ_ERR   = 'Error, close file, check file path or try resaving file as .xls (not xml)'
+
+    # add option for config. file in future 
+    HEADERS = {
+      :oclc      => 19,
+      :date      => 12,
+      :time      => 21,
+      :isbn      => 1,
+      :author    => 4,
+      :title     => 2,
+      :place     => 11,
+      :publisher => 3,
+      :requires  => 10,
+      :format    => 9,
+      :filesize  => 8,
+      :reader    => 14,
+      :title_src => 13,
+      :summary   => 15,
+      :subjects  => 5,
+      :download  => 7,
+      :excerpt   => 16,
+      :cover     => 17,
+      :thumb     => 18,
+    }
+
+    def initialize(metadata_file)
+        begin   
+        @metadata = Spreadsheet.open(metadata_file).worksheet 0
+        @coder    = HTMLEntities.new
+        @records  = []
+        map
+        merge_by_isbn
+        rescue Exception => ex
+            raise READ_ERR
+        end
+    end
+
+    def map
+        @metadata.each do |row|
+            @records << create_record(row)
+        end
+      @records.compact
+    end
+
+    def create_record(data)
+      field = package_data(data)
+      r = field[:format].match(/#{DEF_FORMAT}/) ? EBook.new : EAudioBook.new
+      begin
+        r.make_control_field('001', field[:oclc])
+        r.make_006
+        r.make_007
+        r.make_fixed_field(field[:year], field[:month], field[:day])
+        r.make_data_field('020', ' ', ' ', {'a' => field[:isbn] + ' ' + r.isbn}) unless field[:isbn].empty?
+        r.make_data_field('037', ' ', ' ', {'b' => OD_ORG, 'n' => OD_URL})
+        r.make_data_field('040', ' ', ' ', {'a' => AGENCY, 'c' => AGENCY})
+        r.make_data_field('100', '1', ' ', {'a' => normalize_author(field[:author])})
+        r.make_title(field[:title], field[:author])
+        r.make_publication(field[:place], field[:publisher], field[:year])
+        r.make_physical(field[:hours], field[:minutes])
+        r.make_data_field('306', ' ', ' ', {'a' => field[:hours] + field[:minutes] + field[:seconds]})
+        r.make_data_field('538', ' ', ' ', {'a' => ACCESS})
+        r.make_data_field('538', ' ', ' ', {'a' => 'Requires ' + field[:requires] + '.'})  
+        r.make_data_field('500', ' ', ' ', {'a' => "#{field[:format]} (file size: #{field[:filesize]} MB)."})
+        r.make_data_field('511', '0', ' ', {'a' => "Read by #{field[:reader]}."}) unless field[:reader].empty? 
+        r.make_data_field('520', ' ', ' ', {'a' => field[:summary]}) unless field[:summary].match(/^#+$/)
+        r.make_data_field('500', ' ', ' ', {'a' => "Title from: #{field[:title_src]}."})
+        r.make_data_field('500', ' ', ' ', {'a' => 'Unabridged.'}) if r.is_a? EAudioBook
+        r.make_data_field('500', ' ', ' ', {'a' => "Duration: #{field[:hours]} hr., #{field[:minutes]} min."}) if r.is_a? EAudioBook
+        field[:subjects].each { |s| r.make_data_field('655', ' ', '7', {'a' => @coder.decode(s).strip + '.', '2' => 'local'}) }
+        r.make_data_field('655', ' ', '7', {'a' => r.subject, '2' => 'local'})
+        r.make_data_field('700', '1', ' ', {'a' => normalize_author(field[:reader])})
+        r.make_data_field('856', '4', '0', {'u' => field[:download], 'y' => URL_MSG})
+        r.make_data_field('856', '4', '0', {'u' => field[:excerpt], 'y' => "Excerpt (#{field[:format]})."})
+        r.make_data_field('856', '4', '2', {'u' => field[:cover], 'y' => "<img class=\"scl_mwthumb\" src=\"#{field[:thumb]}\" alt=\"Artwork for this title - #{field[:title].gsub(/[^A-Za-z ]/, '')}\" />"})
+        r.make_data_field('907', ' ', ' ', {'a' => 'ER'})
+        r.make_data_field('991', ' ', ' ', {'a' => DISCLAIM})
+        return r.record
+      rescue Exception => ex
+        puts ex.message
+        nil
+      end
+    end
+
+    def package_data(data)
+      values = {}
+      values[:isbn]             = data[HEADERS[:isbn]]
+      values[:date]             = data[HEADERS[:date]]
+      values[:place]            = data[HEADERS[:place]]
+      values[:publisher]        = data[HEADERS[:publisher]]
+      values[:month]            = ''
+      values[:day]              = ''
+      if values[:date].match(/\d{1,2}\/\d{1,2}\/\d{4}/)
+        month, day, year        = values[:date].split '/'
+        values[:month]          = month
+        values[:day]            = day
+      end
+      values[:year]             = values[:date].match(/\d{4}/).to_s # Fall-back
+      values[:time]             = data[HEADERS[:time]]
+      hr, mn, sc                = values[:time].split ':'
+      values[:hours]            = hr ? hr : ''
+      values[:minutes]          = mn ? mn : ''
+      values[:seconds]          = sc ? sc : ''
+      values[:author]           = @coder.decode(data[HEADERS[:author]])
+      values[:title]            = @coder.decode(data[HEADERS[:title]])
+      values[:title_src]        = data[HEADERS[:title_src]]
+      values[:reader]           = @coder.decode(data[HEADERS[:reader]])
+      values[:requires]         = data[HEADERS[:requires]]
+      values[:format]           = data[HEADERS[:format]]
+      values[:filesize]         = kb_to_mb(data[HEADERS[:filesize]])
+      values[:summary]          = Sanitize.clean(@coder.decode(data[HEADERS[:summary]])).gsub(/\s{2}+/, '').strip
+      values[:subjects]         = data[HEADERS[:subjects]].split ','
+      values[:download]         = data[HEADERS[:download]]
+      values[:excerpt]          = data[HEADERS[:excerpt]]
+      values[:thumb]            = data[HEADERS[:thumb]]
+      values[:cover]            = data[HEADERS[:cover]]
+      values[:oclc]             = data[HEADERS[:oclc]].to_s.empty? ? 'ovr' + make_id(values[:download]) : 'ocn' + data[HEADERS[:oclc]]
+      return values
+    end
+
+    def merge_by_isbn
+        isbns = Hash.new(0)
+        @records.each do |record|
+          isbns[record['020'].value] += 1 if record['020']
+        end
+        isbns.delete_if { |k,v| v < 2 }
+        isbns.keys.each do |isbn|
+          rcds = @records.find_all { |r| r['020']['a'] == isbn if r['020'] }
+          raise 'Found invalid number of duplicate records: ' + isbn unless rcds.size == 2
+          file_note = rcds[1].find { |f| f.tag == '500' and f['a'] =~ /OverDrive (WMA|MP3) Audiobook/ }
+          excerpt   = rcds[1].find { |f| f.tag == '856' and f['y'] =~ /Excerpt/ }
+          if file_note and excerpt
+            rcds[0].fields.insert(rcds[0].fields.index { |f| f.tag == '500' }, file_note)
+            rcds[0].fields.insert(rcds[0].fields.index { |f| f.tag == '856' and f['y'] =~ /Excerpt/ }, excerpt)
+            @records.delete rcds[1]
+          end
+        end
+        @records
+    end
+
+    def make_id(id_string)
+        return id_string[-9..-1].gsub(/\W/, '')
+    end
+
+    def normalize_author(author)
+        return author if author.empty?
+        author   = author.split(',')[0]
+        names    = author.split ' '
+        surname  = names.last + ', '
+        fullname = surname + names[0 .. names.length - 2].join(' ')
+        fullname += '.' unless fullname[-1] == '.'
+        return fullname
+    end
+
+    ##
+    # Quickly turn 325645 {kb} into 318 {mb} etc. + 1 so not 0
+
+    def kb_to_mb(size)
+      return (size.to_f / 1024 + 1).to_i.to_s
+    end
+
+    class ERecord
+
+      GMD      = '[electronic resource]'
+      DATE_ERR = 'Date information not present for fixed field'
+      FIXF_ERR = 'Invalid fixed field created'
+      TITL_ERR = 'Title data is missing for record'
+
+      attr_reader :record
+
+      def initialize
+        @record = MARC::Record.new
+        @ldr = record.leader
+        @ldr[5]  = 'n'
+        @ldr[7]  = 'm'
+        @ldr[17] = 'M'
+        @ldr[18] = 'a'
+        fixed_field = ''
+      end
+
+      def make_control_field(tag, value)
+        return nil if value.empty?
+        @record.append MARC::ControlField.new(tag, value)
+      end
+
+      def make_data_field(tag, ind1, ind2, subfields)
+        s = []
+        subfields.each do |k,v|
+          return nil if v.empty?
+          s << MARC::Subfield.new(k, v)
+        end
+        @record.append MARC::DataField.new(tag, ind1, ind2, *s)
+      end
+
+      def make_fixed_field(year, month, day)
+        raise DATE_ERR if year.empty?
+        fixed_field = @fixed_field
+        unless month.empty? and day.empty?
+          month = '0' + month if month.length == 1
+          day   = '0' + day if day.length == 1
+          fixed_field[0..5] = year[2..3] + month + day
+          fixed_field[7..10] = year
+        else
+          fixed_field[7..10] = year
+        end
+        raise FIXF_ERR unless fixed_field.length == 40
+        make_control_field('008', fixed_field)
+      end
+
+      def make_title(title, sor)
+        raise TITL_ERR if title.empty?
+          t_ind1  = sor.empty? ? '0' : '1'
+          t_ind2  = non_filing_characters title
+          subfields = {}
+          subfields['a'] = title
+          subfields['h'] = GMD + ' /'
+          unless sor.empty?
+            value = sor[-1] == '.' ? "by #{sor}" : "by #{sor}."
+            subfields['c'] = value
+          else
+            subfields['h'].gsub!(/\s+\/$/, '.')
+          end
+          make_data_field('245', t_ind1, t_ind2, subfields)
+      end
+
+      def make_publication(place, publisher, year)
+        return nil if place.empty? or publisher.empty? or year.empty?
+        make_data_field('260', ' ', ' ', {'a' => "#{place} :", 'b' => "#{publisher},", 'c' => "#{year}."})
+      end
+
+      def non_filing_characters(title)
+        return case
+        when title.match(/^The /)
+            '4'
+        when title.match(/^An /)
+            '3'
+        when title.match(/^A /)
+            '2'     
+        else
+            '0'
+        end
+      end
+
+    end
+
+    class EBook < ERecord
+
+      attr_reader :isbn, :subject
+        
+      def initialize
+          super
+          @ldr[6]  = 'a'
+          @fixed_field = '      s        xxu|||| s      0||| eng d'
+          @isbn = '(electronic bk. : OverDrive Electronic Book)'
+          @subject = 'Downloadable ebooks.'
+          # set leader
+      end
+
+      def make_006
+        make_control_field('006', 'm        d        ')
+      end
+
+      def make_007
+        make_control_field('007', 'cr nnu---|||||')
+      end
+
+      def make_physical(*args)
+        make_data_field('300', ' ', ' ', {'a' => "1 online resource."})
+      end
+      
+    end
+
+    class EAudioBook < ERecord
+
+      attr_reader :isbn, :subject
+
+      def initialize
+        super
+        @ldr[6]  = 'i'
+        @fixed_field = '      s        xxunnnn s           eng d'
+        @isbn = '(sound recording : OverDrive Audio Book)'
+        @subject = 'Downloadable audiobooks.'
+        # set leader
+      end
+
+      def make_006
+        make_control_field('006', 'm        h        ')
+      end
+
+      def make_007
+        make_control_field('007', 'sz usnnnnnnned')
+        make_control_field('007', 'cr nna   |||||')
+      end
+
+      def make_physical(hours, minutes)
+        return nil if hours.empty? or minutes.empty?
+        make_data_field('300', ' ', ' ', {'a' => "1 sound file (ca. #{hours} hr., #{minutes} min.) :", 'b' => 'digital.'})
+      end
+      
+    end
+
+end

data/test/test_overdrive_metadata.rb

@@ -0,0 +1,16 @@
+require "shoulda"
+require "overdrive_metadata"
+
+class TestOverdriveMetadata < Test::Unit::TestCase
+  
+  context "Creating Overdrive records" do
+  
+  	setup do 
+  		@o = OverdriveMetadata.new('raw/test.xls')
+  	end
+
+    # Write some tests ...
+
+  end
+
+end

metadata

@@ -0,0 +1,75 @@
+--- !ruby/object:Gem::Specification
+name: overdrive_metadata
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Mark Cooper
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-11-21 00:00:00.000000000 -08:00
+default_executable: 
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: hoe
+  requirement: &10533756 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.12'
+  type: :development
+  prerelease: false
+  version_requirements: *10533756
+description: Generate marc records from Overdrive provided metadata spreadsheets.
+email:
+- markchristophercooper@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- History.txt
+- Manifest.txt
+- README.txt
+files:
+- .autotest
+- History.txt
+- Manifest.txt
+- README.txt
+- Rakefile
+- lib/overdrive_metadata.rb
+- raw/test.xls
+- test/test_overdrive_metadata.rb
+- .gemtest
+has_rdoc: true
+homepage: ! 'http://www.libcode.net
+
+'
+licenses: []
+post_install_message: 
+rdoc_options:
+- --main
+- README.txt
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: overdrive_metadata
+rubygems_version: 1.5.2
+signing_key: 
+specification_version: 3
+summary: Generate marc records from Overdrive provided metadata spreadsheets.
+test_files:
+- test/test_overdrive_metadata.rb