mead 0.0.5

data/lib/mead/ead.rb

@@ -0,0 +1,215 @@
+module Mead
+  class Ead
+    # factor out :baseurl, :file and :url into an options object?
+    attr_accessor :containers, :ead, :baseurl, :file, :url, :doc, :eadid, :series_present,
+      :c01s_series_answer
+
+    # options include :file and :base_url
+    def initialize(opts={})
+      @eadid = opts[:eadid] || nil
+      @file    = opts[:file] || nil
+      @baseurl = opts[:baseurl] || nil
+      @url     = opts[:url] || nil
+      @containers = []
+
+      get_ead
+      find_eadid unless @eadid
+      crawl_for_containers
+    end
+
+    def get_ead
+      if @eadid.nil? and @url.nil? and @file.nil? and @baseurl
+        raise 'Cannot get EAD based on params.'
+      end
+      if @file and @file.is_a? File
+        @file.rewind if @file.eof?
+        @ead = @file.read
+      elsif @url
+        @ead = open(@url).read
+      elsif @baseurl
+        @ead = open(File.join(@baseurl, @eadid + '.xml')).read
+      end
+      @doc = Nokogiri::XML(@ead)
+    end
+
+    def find_eadid
+      begin
+        @eadid = @doc.xpath('//xmlns:eadid').first.text
+      rescue => e
+        raise 'Need an eadid and none has been given and it cannot be found in the EAD XML.'
+      end
+    end
+
+    def crawl_for_containers
+      c01s.each_with_index do |c, i|
+        dids = c.xpath('.//xmlns:container').map{|c| c.parent}.uniq
+        #c.xpath('xmlns:c02/xmlns:did').map do |did|
+        dids.map do |did|
+          info = {}
+          if c01s_series?
+            info[:series] = i + 1 # if all the c01s are at the file level this fails
+          else
+            info[:series] = 0
+          end
+          info[:mead] = create_mead(did, i)
+          info[:title] = concat_title(did)
+          # FIXME
+          info[:containers] = text_containers(did)
+          @containers << info
+        end
+      end
+    end
+    
+    def text_containers(did)
+      did.xpath('xmlns:container').map do |container|
+        text = ''
+        text << container.attribute('type').text + ' ' if container.attribute('type')
+        text << container.text if container.text
+        text
+      end
+    end
+    
+    def c01s
+      @doc.xpath('//xmlns:dsc/xmlns:c01')
+    end
+    
+    def c01s_series?
+      @c01s_series_answer ||= c01s.length == series_c01s.length
+    end
+    
+    def series_c01s
+      @doc.xpath("//xmlns:dsc/xmlns:c01[@level='series']")
+    end
+
+    def create_mead(did, i)
+      mead = [@eadid.dup]
+      if c01s_series?
+        mead << "%03d" % (i + 1) #series
+      else
+        mead << '001'
+      end
+      begin
+        mead << specific_containers(did)
+      rescue
+        return @mead = mead.flatten.join('-')        
+      end
+      mead << '001' # stub for first record
+      @mead = mead.flatten.join('-')
+    end
+
+    def concat_title(did)
+      title = ''
+      title << did.xpath('xmlns:unittitle').text if did.xpath('xmlns:unittitle')
+      if did.xpath('xmlns:unittitle') and did.xpath('xmlns:unitdate') and !did.xpath('xmlns:unitdate').text.empty?
+        title << ', ' << did.xpath('xmlns:unitdate').text
+      end
+      if did.xpath('xmlns:unitid') and !did.xpath('xmlns:unitid').text.empty?
+        title << ' (' + did.xpath('xmlns:unitid').text + ')'
+      end
+      title
+    end
+
+    def specific_containers(did)
+      containers = did.xpath('xmlns:container')
+      container_values = []
+      if containers.length == 1
+        container_values << make_box(did.xpath('xmlns:container')[0])
+        container_values << '000'
+      elsif containers.length == 2
+        container_values << make_box(did.xpath('xmlns:container')[0])
+        container_values << make_box(did.xpath('xmlns:container')[1],3)
+      elsif containers.length > 2
+        raise "I can't create a mead identifier with more than 2 containers in a did!"
+      else
+        raise "Do we really have zero containers?!"
+      end
+      return container_values
+    end
+
+    def make_box(container, padding=4)
+      # FIXME: pad based on first part of range for folder +++
+      padder = "%0" + padding.to_s + 's'
+      text = (padder % container.text).gsub(' ','0').gsub('.','_').gsub('-', '~')
+      container_type(container) + text
+    end
+
+    def container_type(container)
+      match =''
+      CONTAINER_MAPPING.each do |k,v|
+        if container.attribute('type').text == v or
+            container.attribute('type').text.downcase == v
+          match = k
+        end
+      end
+      match
+    end
+
+    def to_csv
+      Mead::Ead.to_csv(self.containers)
+    end
+
+    def self.to_csv(container_list)
+      if CSV.const_defined? :Reader
+        csv_class = FasterCSV # old CSV was loaded
+      else
+        csv_class = CSV # use CSV from 1.9
+      end
+      csv_string = csv_class.generate do |csv|
+        # FIXME
+        csv << ['mead','title','series', 'containers']
+        #csv << ['mead','title','series']
+        container_list.each do |container|
+          csv << [container[:mead], container[:title], container[:series], container[:containers].join(', ')]
+          #csv << [container[:mead], container[:title], container[:series]]
+        end
+      end
+    end
+
+    def valid?
+      if unique_meads.length == @containers.length
+        if short_meads?
+          false
+        else
+          true
+        end
+      else
+        false
+      end
+    end
+    
+    def unique_meads
+      @containers.collect{|container| container[:mead]}.uniq
+    end
+    
+    def long_meads
+      unique_meads.select{|m| m.split('-').length > 2}
+    end
+    
+    def short_meads
+      unique_meads.select{|m| m.split('-').length <= 2}
+    end
+    
+    def short_meads?
+      if unique_meads.length == long_meads.length
+        false
+      else
+        true
+      end
+    end
+
+    def invalid
+      duplicates = dups
+      @containers.select{|container| duplicates.include?(container[:mead])}
+    end
+
+    def dups
+      meads.inject({}) {|h,v| h[v]=h[v].to_i+1; h}.reject{|k,v| v==1}.keys.sort
+    end
+
+    def meads
+      @containers.collect{|container| container[:mead]}
+    end
+
+  end
+end
+

data/lib/mead/ead_validator.rb

@@ -0,0 +1,46 @@
+module Mead
+  class EadValidator
+    attr_accessor :directory, :valid, :invalid, :invalid_full
+
+    # Creates a new EadValidator when given the path to a directory as a String
+    def initialize(directory)
+      @directory = directory
+      @valid = []
+      @invalid = []
+      @invalid_full = []
+    end
+
+    def validate!
+      files = Dir.glob(File.join(@directory, '*.xml')).sort
+      threads = []
+      files.map do |path|
+        threads << Thread.new(path) do |path_t|
+          eadid = File.basename(path_t, '.xml')
+          begin
+            ead = Mead::Ead.new({:file => File.open(path_t), :eadid => eadid})
+          rescue => e
+            record_invalid(eadid, ead, e)
+            next
+          end
+          if ead.valid?
+            @valid << eadid
+          else
+            record_invalid(eadid, ead)
+          end
+        end
+      end
+      threads.each { |thread|  thread.join }
+      metadata
+    end
+
+    def record_invalid(eadid, ead, error=nil)
+      @invalid << eadid
+      @invalid_full << {:eadid => eadid, :error => error, :dups => ead.dups, :containers => ead.invalid}
+    end
+
+    def metadata
+      {:valid => @valid.sort, :invalid => @invalid.sort}
+    end
+  end
+end
+

data/lib/mead/extractor.rb

@@ -0,0 +1,198 @@
+module Mead
+  class Extractor
+
+    attr_accessor :mead, :dsc, :series, :doc, :ead_location, :stack, :node
+
+    # a stack contains the data (unittitle, unitdate) from the identifier's
+    # container all the way through to parent containers. The order is from
+    # most specific to least specific
+    def initialize(mead)
+      @stack = []
+      get_mead_obj(mead)
+      self
+    end
+
+    def extract
+      get_ead_location
+      eadxml = get_eadxml
+      @doc = Nokogiri::XML(eadxml)
+      do_extraction
+      @mead.metadata = @stack # make sure the metadata always gets cached to the identifier
+      return @stack
+    end
+
+    private
+
+    def get_mead_obj(mead)
+      if mead.is_a? Mead::Identifier
+        @mead = mead
+      else
+        raise "is not a Mead::Identifier"
+      end
+    end
+
+    def get_ead_location
+      if mead.ead_location
+        @ead_location = mead.ead_location
+      else
+        raise 'Cannot extract because no Ead location defined in Mead::Identifier.'
+      end
+    end
+
+    def do_extraction
+      get_dsc
+      get_series
+      find_node
+      push_to_stack(@node)
+    end
+
+    def push_to_stack(node)
+      return nil if node.nil?
+      additional_did = {:unittitle => container_field('unittitle', node),
+        :unitdate => container_field('unitdate', node),
+        :level => container_level(node),
+        :unitid => container_field('unitid', node)
+      }
+      did_location_text = did_location(node)
+      additional_did[:item_location] = did_location_text if did_location_text
+      
+      add_containers(additional_did, node)
+      
+      if additional_did[:level] == 'series'
+        additional_did[:series_number] = series_number(node)
+      end
+      if @stack.last == additional_did
+        return
+      end
+      @stack << additional_did
+      if !node.parent.parent.xpath('xmlns:did').empty?
+        push_to_stack(node.parent.parent.xpath('xmlns:did')[0])
+      end
+    end
+
+    def add_containers(hash, node)
+      if !node.xpath('./xmlns:container').empty?
+        hash[:containers] = []
+        node.xpath('./xmlns:container').each do |container|
+          c = Mead::Container.new
+          c.type = container.attribute('type').text if container.attribute('type')
+          c.label = container.attribute('label').text if container.attribute('label')
+          c.text = container.text if !container.text.empty?
+          hash[:containers] << c
+        end
+      end
+    end
+
+    def did_location(did)
+      location = []
+      did.xpath('./xmlns:container').each do |container|
+        location << container.attribute('type').text + ' ' + container.text
+      end
+      unless location.empty?
+        location.join(', ')
+      end
+    end
+
+    def get_series
+      c01_series = @dsc.xpath(".//xmlns:c01[@level='series']")
+      if c01_series and !c01_series.empty?
+        c01_series.each_with_index do |c01, i|
+          if mead.series.to_i == i + 1
+            @series = c01
+          end
+        end
+      else
+        @series = @dsc
+      end
+    end
+    
+    def folder_types
+      types = "@type='#{@mead.folder[:type]}' or @type='#{@mead.folder[:type].capitalize}'"
+      if @mead.folder[:type] == 'folder'
+        types << " or @type='envelope' or @type='Envelope'"
+      end
+      types
+    end
+
+    def find_node(folder=true)
+      #dsc_dids = series.xpath('.//xmlns:did')
+      if @mead.container[:type]    
+        container_set_xpath = ".//xmlns:container[text()='#{@mead.container[:number]}' and (@type='#{@mead.container[:type]}' or @type='#{@mead.container[:type].capitalize}')]"
+        if folder and @mead.folder
+          container_set_xpath << "/../xmlns:container[text()='#{@mead.folder[:number]}' and (#{folder_types})]"
+        end
+        containers = series.xpath(container_set_xpath)
+        #matching_dids
+        if containers.length > 1
+          raise "too many matching nodes!"
+        elsif containers.length == 0
+          # Second chance to handle legacy identifiers where a blank folder was given as 001
+          if @mead.folder[:number] == '1'
+            #@mead.folder = nil #TODO: check do 000 folders get automatically turned to nil when the mead is created?
+            find_node(false)
+          else
+            raise "no matching dids!"
+          end
+        else
+          @node = containers[0].parent
+        end
+      else
+        return nil
+      end
+    end
+
+    def container_field(field, node)
+      xpath = 'xmlns:' + field
+      if node.xpath(xpath)
+        text = node.xpath(xpath).text
+        if text.nil? or text.empty?
+          return nil
+        else
+          return text
+        end
+      else
+        nil
+      end
+    end
+
+    def container_level(node)
+      if node.parent['level']
+        node.parent['level']
+      else
+        nil
+      end
+    end
+
+    def series_number(node)
+      parent_node = node.parent
+      siblings = node.document.xpath('//xmlns:c01')
+      length = siblings.length
+      index = siblings.index(parent_node) + 1
+      index
+    end
+
+    def get_dsc
+      @dsc = @doc.xpath('//xmlns:dsc')
+    end
+
+    def get_eadxml
+      tries = 5
+      begin
+        if @ead_location.respond_to? :read
+          @ead_location.read
+        else
+          return open(@ead_location).read
+        end
+      rescue => e
+        tries -= 1
+        if tries > 0
+          retry
+        else
+           raise "Could not get record by eadid! " + e.inspect
+        end
+      end
+    end
+
+  end
+end
+

data/lib/mead/identifier.rb

@@ -0,0 +1,112 @@
+module Mead
+  class Identifier
+
+    attr_accessor :mead, :eadid, :series, :container, :folder, :sequence, :page, 
+      :ead_location, :metadata
+    include Mead::Validations
+    validates_format_of_mead
+    validates_presence_of_mead
+    validates_numericality_of_mead :sequence, :page
+
+    # If a location is given then extraction can take place
+    def initialize(mead, ead_location=nil)
+      @mead = mead
+      @metadata = nil
+      parse_mead 'eadid', 'series', 'container', 'folder', 'sequence'
+      @ead_location = parse_ead_location(ead_location)
+      split_container
+      split_folder
+      split_page
+      clean_zeros 'series', 'sequence', 'page'      
+      self
+    end
+
+    def parse_mead(*args)
+      parts = @mead.split('-')
+      args.each_with_index do |field, i|
+        instance_variable_set('@' + field, parts[i])
+      end    
+    end
+
+    def split_container
+      type = CONTAINER_MAPPING[ @container[0,2] ]
+      number = strip_zeros(container_number_transforms(@container[2,10]))
+      @container = {:type=> type, :number=> number}
+    end
+    
+    def split_folder
+      if CONTAINER_MAPPING.keys.include?(@folder[0,2])
+        type = CONTAINER_MAPPING[ @folder[0,2] ] 
+        number = strip_zeros(container_number_transforms(@folder[2,10]))
+      else
+        type = 'folder'
+        number = strip_zeros(container_number_transforms(@folder))
+      end
+      if number.nil? or (number and number.empty?)
+        @folder = nil
+      else
+        @folder = {:type=> type, :number=> number}
+      end      
+    end
+    
+    def container_number_transforms(string)
+      string.gsub('_','.').gsub('~', '-').gsub(/^0*/,'')
+    end
+
+    def clean_zeros(*args)
+      args.each do |field|
+        instance_var = instance_variable_get('@' + field)
+        if instance_var
+          cleaned_value = strip_zeros(instance_var)
+          instance_variable_set('@' + field, cleaned_value)
+        end
+      end
+    end
+
+    def strip_zeros(num)
+      num.sub(/^0+/,'')
+    end
+    
+    def split_page
+      @sequence, @page = sequence.split('_')
+    end
+    
+    def parse_ead_location(loc)
+      return nil if loc.nil?
+      if loc
+        if loc.is_a? File 
+          loc.rewind if loc.eof?
+          @ead_location = loc
+        elsif loc.include?('http://')
+          if loc.include?(@eadid)
+            @ead_location = loc
+          else
+            @ead_location = File.join(loc, @eadid + '.xml')
+          end
+        end
+      end
+    end
+    
+    def extract
+      @metadata = Mead::Extractor.new(self).extract
+      self
+    end
+    
+#    def ead_has_series?
+#      if series > 1
+#        true
+#      else
+#        if 
+#        false
+#      end
+#    end
+
+#    def replace_underscores(*args)
+#      args.each do |field|
+#        value = instance_variable_get('@' + field).gsub('_', '.')
+#        instance_variable_set('@' + field, value)
+#      end
+#    end
+
+  end
+end

data/lib/mead/trollop.rb

@@ -0,0 +1,38 @@
+module Mead
+  module TrollopOptions
+    def check_options(opts)
+      number_of_get_methods = [:baseurl, :url, :file].inject(0) do |memo, option| 
+        temp_memo = memo
+        temp_memo += 1 if opts[option]
+        temp_memo
+      end
+      if number_of_get_methods > 1
+        Trollop::die 'Must specify ONLY one way to get the Ead XML' 
+      elsif number_of_get_methods == 0
+        Trollop::die 'Must specify at least one way to get the Ead XML' 
+      end
+    end
+    
+    def get_location_options(opts)
+      location_options = {}
+      if opts[:baseurl]
+        location_options[:baseurl] = opts[:baseurl]
+      elsif opts[:url]
+        location_options[:url] = opts[:url]
+      elsif opts[:file]
+        location_options[:file] = File.open(opts[:file])
+      end
+      location_options
+    end
+    
+    def get_location(opts)
+      if opts[:baseurl]
+        opts[:baseurl]
+      elsif opts[:url]
+        opts[:url]
+      elsif opts[:file]
+        File.open(opts[:file])
+      end
+    end
+  end
+end