high_level_browse 0.1.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 286d0ce64d0d9e8dffa58b716f111d086310654d
-  data.tar.gz: 2a13aad07ee29e47b0bcc00f4ba16740491e9bfd
+SHA256:
+  metadata.gz: 5eae24ef8906dfb25f1f949b4b29b69a33834dc63a7b0d96cd356324b14a79df
+  data.tar.gz: 919434516a5098b1c4f9766434c68b7b7e6960128a0eb59abc7e442da520de5e
 SHA512:
-  metadata.gz: 9960852abc0686da303da11c8ead326df0ea7e7df89432962f7d1353e62350afbc7a3ad556d1beecfe6cce816c1bf654ce4bdee78bb195caefdb08caeb67b7cf
-  data.tar.gz: 3d29b51feb0bd70c37eea28248eff4f3dccd8a38cdb23617be8998fdfa821e392743d0f87c25e9f233ee326b4097072ca63d3d04bcdf5c21216ec43a96ecae04
+  metadata.gz: 5aa8cb1cb8472c788c1def8efff185a8fc16957ef97450178687941f6dc5a03444f6d85b1c25f601aa34eef02ed51e8ab7475c9d067e2db35aa94118d319a7dc
+  data.tar.gz: 390752555d91aec5060be2a34097b9cc6ca664b22c44ae6b29a989e9cb766bc45b83599310d2fec2ee79fe3155678c41fc52c24f6a3658692ab7b2ac32bb513d

data/CHANGELOG.md

@@ -0,0 +1,13 @@
+# High Level Browse
+
+## 1.0.0
+
+* New normalization algorithm.
+* Because the normalization algorithm has changed, the on-disk file
+  (`hlb.json.gz`) needs to be re-generated, because the old normalization
+  form and the new one won't match each other.
+
+0.2.0
+
+* First real release
+

data/bin/hlb

@@ -28,7 +28,7 @@ require 'tmpdir'
 filename = HighLevelBrowse::DB::FILENAME
 dir      = Dir.tmpdir()
 fullpath = File.join(dir, filename)
-
+puts fullpath
 hlb = if File.exist?(fullpath)
         HighLevelBrowse.load(dir: dir)
       else

data/bin/test_marc_file_for_hlb

@@ -7,7 +7,6 @@ end
 
 require 'marc'
 require 'high_level_browse'
-require 'lcsort'
 require 'tmpdir'
 
 
@@ -38,7 +37,7 @@ Counter = Struct.new(:count, :invalid, :found, :notfound, :hlb) do
 
 
   def check_cn(cn)
-    normalized = Lcsort.normalize(cn)
+    normalized = HighLevelBrowse::CallNumberRange.callnumber_normalize(cn)
     return :invalid if normalized.nil?
     cats = hlb[cn]
     if cats.empty?

data/high_level_browse.gemspec

@@ -17,10 +17,9 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency 'oga', '~> 2.1'
-  spec.add_dependency 'lcsort'
+  spec.add_dependency 'nokogiri', '~>1.0'
 
-  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "bundler", '~>2.0'
   spec.add_development_dependency "rake"
   spec.add_development_dependency "minitest"
 end

data/lib/high_level_browse/call_number_range.rb

@@ -1,21 +1,25 @@
-require 'lcsort'
 require 'high_level_browse/range_tree'
 
-
 # An efficient set of CallNumberRanges from which to get topics
 class HighLevelBrowse::CallNumberRangeSet < HighLevelBrowse::RangeTree
 
+  ANY_DIGIT = /\d/.freeze
+
+  def has_digits(str)
+    ANY_DIGIT.match?(str)
+  end
 
   # Returns the array of topic arrays for the given LC string
   # @param [String] raw_lc A raw LC string (eg., 'qa 112.3 .A4 1990')
   # @return [Array<Array<String>>] Arrays of topic labels
   def topics_for(raw_lc)
-    normalized = Lcsort.normalize(HighLevelBrowse::CallNumberRange.preprocess(raw_lc))
+    normalized = ::HighLevelBrowse::CallNumberRange.callnumber_normalize(raw_lc)
     self.search(normalized).map(&:topic_array).uniq
+  rescue => e
+    require 'pry'; binding.pry
   end
 end
 
-
 # A callnumber-range keeps track of the original begin/end
 # strings as well as the normalized versions, and can be
 # serialized to JSON
@@ -25,7 +29,6 @@ class HighLevelBrowse::CallNumberRange
 
   attr_reader :min, :max, :min_raw, :max_raw, :firstletter
 
-
   attr_accessor :topic_array, :redundant
 
   SPACE_OR_PUNCT = /\A[\s\p{Punct}]*(.*?)[\s\p{Punct}]*\Z/
@@ -42,27 +45,71 @@ class HighLevelBrowse::CallNumberRange
   def self.force_break_between_digit_and_letter(str)
     str.gsub(DIGIT_TO_LETTER, '\1 \2')
   end
+
   # @nodoc
   # Preprocess the string, removing spaces/punctuation off the end
   # and forcing a space where there's a digit->letter transition
-  def self.preprocess(str)
-    str ||= ''
-    force_break_between_digit_and_letter(
-        strip_spaces_and_punct(str)
-    )
+  # def self.preprocess(str)
+  #   str ||= ''
+  #   force_break_between_digit_and_letter(strip_spaces_and_punct(str)
+  #   )
+  # end
+
+  # Normalize the callnumber in a slightly more sane way
+  # @param [String] cn The raw callnumber to normalize
+  CN = /\A\s*(?<letters>\p{L}{1,3})\s*(?<digits>\d{1,5}(?!\d))(?:\.(?<decimals>\d+))?(?<rest>.*)\Z/.freeze
+
+  def self.callnumber_normalize(cs_str)
+    return nil if cs_str.nil?
+
+    cs_str = cs_str.downcase
+    return cs_str if /\A\s*\p{L}{1,3}+\s*\Z/.match? cs_str # just letters
+
+    m = CN.match(cs_str)
+    return nil unless m
+
+    digits = m[:digits].size.to_s + m[:digits]
+    decimals = m[:decimals] ? "." + m[:decimals] : ""
+    rest = cleanup_freetext(m[:rest])
+    clean = m[:letters] + digits + decimals + " " + rest
+    clean.strip.gsub(/\s+/, ' ')
   end
 
+  # @param [String] str String to clean up
+  def self.cleanup_freetext(str)
+    return "" if str.nil?
+
+    s = str.strip
+    return s if s == ""
+
+    s = replace_dot_before_letter_with_space(s)
+    s = remove_dots_between_letters(s)
+    s = force_space_between_digit_and_letter(s)
+    s.strip.gsub(/\s+/, ' ')
+  end
+
+  def self.replace_dot_before_letter_with_space(s)
+    s.gsub /\.(\p{L})/, '\\1'
+  end
+
+  # @param [String] str
+  def self.remove_dots_between_letters(str)
+    str.gsub(/(\p{L})\.(\p{L})/, '\\1\\2')
+  end
+
+  def self.force_space_between_digit_and_letter(s)
+    s.gsub(/(\d)(\p{L})/, '\\1 \\2')
+  end
 
   def initialize(min:, max:, topic_array:)
-    @illegal     = false
-    @redundant   = false
-    self.min     = self.class.preprocess(min)
-    self.max     = self.class.preprocess(max)
+    @illegal = false
+    @redundant = false
+    self.min = min
+    self.max = max
     @topic_array = topic_array
     @firstletter = self.min[0] unless @illegal
   end
 
-
   # Compare based on @min, then end
   # @param [CallNumberRange] o the range to compare to
   def <=>(o)
@@ -74,31 +121,25 @@ class HighLevelBrowse::CallNumberRange
   end
 
   def reconstitute(min, max, min_raw, max_raw, firstletter, topic_array)
-    @min         = min
-    @max         = max
-    @min_raw     = min_raw
-    @max_raw     = max_raw
+    @min = min
+    @max = max
+    @min_raw = min_raw
+    @max_raw = max_raw
     @firstletter = firstletter
     @topic_array = topic_array
   end
 
-
   # Two ranges are equal if their @min, @max, and topic array
   # are all the same
   # @param [CallNumberRange] o the range to compare to
   def ==(other)
-    @min == other.min and
-        @max == other.max and
-        @topic_array == other.topic_array
+    @min == other.min and @max == other.max and @topic_array == other.topic_array
   end
 
-
   # @nodoc
   # JSON roundtrip
   def to_json(*a)
-    {
-        'json_class' => self.class.name,
-        'data'       => [@min, @max, @min_raw, @max_raw, @firstletter, @topic_array]
+    {'json_class' => self.class.name, 'data' => [@min, @max, @min_raw, @max_raw, @firstletter, @topic_array]
     }.to_json(*a)
   end
 
@@ -109,29 +150,26 @@ class HighLevelBrowse::CallNumberRange
     cnr
   end
 
-
   # In both @min= and end=, we also rescue any parsing errors
   # and simply set the @illegal flag so we can use it later on.
   def min=(x)
-    @min_raw     = x
-    possible_min = Lcsort.normalize(x)
+    @min_raw = x
+    possible_min = self.class.callnumber_normalize(x)
     if possible_min.nil? # didn't normalize
       @illegal = true
       nil
-    else
-      @min = possible_min
+    else @min = possible_min
     end
   end
 
   # Same as start. Set the illegal flag if we get an error
   def max=(x)
-    @max_raw     = x
-    possible_max = Lcsort.normalize(x)
+    @max_raw = x
+    possible_max = self.class.callnumber_normalize(x)
     if possible_max.nil? # didn't normalize
       @illegal = true
       nil
-    else
-      @max = possible_max + '~' # add a tilde to make it a true endpoint
+    else @max = possible_max + '~' # add a tilde to make it a true endpoint
     end
   end
 
@@ -139,7 +177,6 @@ class HighLevelBrowse::CallNumberRange
     @illegal
   end
 
-
   def surrounds(other)
     @min <= other.min and @max >= other.max
   end

data/lib/high_level_browse/db.rb

@@ -1,4 +1,4 @@
-require 'oga'
+require 'nokogiri'
 require 'high_level_browse/call_number_range'
 require 'zlib'
 require 'json'
@@ -13,7 +13,7 @@ class HighLevelBrowse::DB
   # database with an efficient structure for querying
   # @param [Array<HighLevelBrowse::CallNumberRange>] array_of_ranges
   def initialize(array_of_ranges)
-    @all    = array_of_ranges
+    @all = array_of_ranges
     @ranges = self.create_letter_indexed_ranges(@all)
   end
 
@@ -22,8 +22,8 @@ class HighLevelBrowse::DB
   # @private
   def create_letter_indexed_ranges(all)
     bins = {}
-    ('A'..'Z').each do |letter|
-      cnrs         = all.find_all {|x| x.firstletter == letter}
+    ('a'..'z').each do |letter|
+      cnrs = all.find_all { |x| x.firstletter == letter }
       bins[letter] = HighLevelBrowse::CallNumberRangeSet.new(cnrs)
     end
     bins
@@ -40,16 +40,17 @@ class HighLevelBrowse::DB
   # @return [Array<Array>] A (possibly empty) array of arrays of topics
   def topics(*raw_callnumber_strings)
     raw_callnumber_strings.reduce([]) do |acc, raw_callnumber_string|
-      firstletter = raw_callnumber_string.strip.upcase[0]
+      firstletter = if raw_callnumber_string.nil?
+                      nil
+                    else raw_callnumber_string.to_s.strip.downcase[0]
+                    end
       if @ranges.has_key? firstletter
         acc + @ranges[firstletter].topics_for(raw_callnumber_string)
-      else
-        acc
+      else acc
       end
     end.uniq
   end
 
-
   alias_method :[], :topics
 
   # Create a new object from a string with the XML
@@ -58,12 +59,11 @@ class HighLevelBrowse::DB
   #    (e.g., from 'https://www.lib.umich.edu/browse/categories/xml.php')
   # @return [DB]
   def self.new_from_xml(xml)
-    oga_doc_root         = Oga.parse_xml(xml)
-    simple_array_of_cnrs = cnrs_within_oga_node(node: oga_doc_root)
+    noko_doc_root = Nokogiri::XML(xml)
+    simple_array_of_cnrs = cnrs_within_noko_node(node: noko_doc_root)
     self.new(simple_array_of_cnrs).freeze
   end
 
-
   # Save to disk
   # @param [String] dir The directory where the hlb.json.gz file will be saved
   # @return [DB] The loaded database
@@ -73,7 +73,6 @@ class HighLevelBrowse::DB
     end
   end
 
-
   # Load from disk
   # @param [String] dir The directory where the hlb.json.gz file is located
   # @return [DB] The loaded database
@@ -81,12 +80,11 @@ class HighLevelBrowse::DB
     simple_array_of_cnrs = Zlib::GzipReader.open(File.join(dir, FILENAME)) do |infile|
       JSON.load(infile.read).to_a
     end
-    db                   = self.new(simple_array_of_cnrs)
+    db = self.new(simple_array_of_cnrs)
     db.freeze
     db
   end
 
-
   # Freeze everything
   # @return [DB] the frozen db
   def freeze
@@ -102,49 +100,44 @@ class HighLevelBrowse::DB
   #  * what the current topics are ([level1, level2])
   # Get all the call numbers assocaited with the topic represented by the given node,
   # as well as all the children of the given node, and send it back as a big ol' array
-  # @param [Oga::Node] node A node of the parsed HLB XML file
+  # @param [Nokogiri::XML::Node] node A node of the parsed HLB XML file
   # @param [Array<String>] decendent_xpaths A list of xpaths to the decendents of this node
   # @param [Array<String>] topic_array An array with all levels of the topics associated with this node
   # @return [Array<HighLevelBrowse::CallNumberRange>]
-  def self.cnrs_within_oga_node(node:, decendent_xpaths: ['/hlb/subject', 'topic', 'sub-topic'], topic_array: [])
+  def self.cnrs_within_noko_node(node:, decendent_xpaths: ['/hlb/subject', 'topic', 'sub-topic'], topic_array: [])
     if decendent_xpaths.empty?
       [] # base case -- we're as low as we're going to go
-    else
-      current_xpath_component = decendent_xpaths[0]
-      new_xpath               = decendent_xpaths[1..-1]
-      new_topic               = topic_array.dup
-      new_topic.push node.get(:name) unless node == node.root_node # skip the root
-      cnrs = []
-      # For each sub-component, get both the call-number-ranges (cnrs) assocaited
-      # with this level, as well as recusively getting from all the children
-      node.xpath(current_xpath_component).each do |c|
-        cnrs += call_numbers_list_from_leaves(node: c, topic_array: new_topic)
-        cnrs += cnrs_within_oga_node(node: c, decendent_xpaths: new_xpath, topic_array: new_topic)
-      end
-      cnrs
+    else current_xpath_component = decendent_xpaths[0]
+    new_xpath = decendent_xpaths[1..-1]
+    new_topic = topic_array.dup
+    new_topic.push node[:name] unless node == node.document # skip the root
+    cnrs = []
+    # For each sub-component, get both the call-number-ranges (cnrs) assocaited
+    # with this level, as well as recusively getting from all the children
+    node.xpath(current_xpath_component).each do |c|
+      cnrs += call_numbers_list_from_leaves(node: c, topic_array: new_topic)
+      cnrs += cnrs_within_noko_node(node: c, decendent_xpaths: new_xpath, topic_array: new_topic)
+    end
+    cnrs
     end
   end
 
-
   # Given a second-to-lowest-level node, get its topic and
   # extract call number ranges from its children
   def self.call_numbers_list_from_leaves(node:, topic_array:)
-    cnrs      = []
-    new_topic = topic_array.dup.push node.get(:name)
+    cnrs = []
+    new_topic = topic_array.dup.push node[:name]
     node.xpath('call-numbers').each do |cn_node|
-      min = cn_node.get(:start)
-      max = cn_node.get(:end)
+      min = cn_node[:start]
+      max = cn_node[:end]
 
       new_cnr = HighLevelBrowse::CallNumberRange.new(min: min, max: max, topic_array: new_topic)
       if new_cnr.illegal?
-        # do some sort of logging
-      else
-        cnrs.push new_cnr
+        # do some sort of logging else cnrs.push new_cnr
       end
     end
     cnrs
 
   end
 
-
 end

data/lib/high_level_browse/range_tree.rb

@@ -48,8 +48,8 @@ module HighLevelBrowse
     end
 
     def search(range, limit: Float::INFINITY)
+      return [] if range.nil?
       range = range.is_a?(Range) ? range : (range..range)
-
       result = []
       RangeTree.search_helper(range, @root, result, limit)
 
@@ -58,7 +58,6 @@ module HighLevelBrowse
 
     def self.search_helper(q, root, result, limit)
       return if root.nil?
-
       # Visit left child?
       if (l = root.left) and l.max and q.min and \
         not l.max < q.min # The interesting part.
@@ -70,6 +69,10 @@ module HighLevelBrowse
       # point of checking, there wasn't added too many, but after left child has
       # been checked, we might hit the limit and then, "this" will add one as
       # well.
+      #
+      # (I'm leaving the above paragraph intact as a reminder to myself to
+      # read things over and make sure they're, you know, actual English. At this point
+      # in 2022 I have no idea what the heck I was saying.
 
       # Add root?
       result << root.range if RangeTree.ranges_intersect?(q, root.range)

data/lib/high_level_browse/version.rb

@@ -1,3 +1,3 @@
   module HighLevelBrowse
-    VERSION = "0.1.0"
+    VERSION = "1.0.0"
   end

data/test/minitest_helper.rb

@@ -5,7 +5,6 @@ $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
 
 verbose = $VERBOSE
 $VERBOSE = nil
-require 'oga'
 require 'minitest'
 require 'minitest/spec'
 require 'minitest/autorun'

data/test/test_high_level_browse.rb

@@ -13,15 +13,3 @@ describe "loads" do
   end
 end
 
-describe "Works the same as before" do
-  it "gets the same output for 30k randomly chosen call numbers" do
-    h = HighLevelBrowse.fetch_and_save(dir: TESTDIR)
-    JSON.load(File.open(File.join(TESTDIR, '30k_random_old_mappings.json'))).each do |rec|
-      cn = rec['cn'].strip
-      newcats = h[cn]
-      next if rec['jar'].empty?
-      assert_equal [cn, rec['jar'].sort], [rec['cn'], newcats.sort]
-    end
-
-  end
-end

metadata

@@ -1,57 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: high_level_browse
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Bill Dueber
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-06-02 00:00:00.000000000 Z
+date: 2022-05-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: oga
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
-- !ruby/object:Gem::Dependency
-  name: lcsort
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -80,7 +66,7 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+description:
 email:
 - bill@dueber.com
 executables:
@@ -92,12 +78,11 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
-- bench/bench.rb
-- bench/hlb.json.gz
 - bin/fetch_new_hlb
 - bin/hlb
 - bin/test_marc_file_for_hlb
@@ -113,7 +98,7 @@ homepage: ''
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -128,9 +113,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.8
-signing_key: 
+rubygems_version: 3.3.3
+signing_key:
 specification_version: 4
 summary: Map LC call numbers to academic categories.
 test_files:

data/bench/bench.rb

@@ -1,57 +0,0 @@
-require 'benchmark/ips'
-$:.unshift '../lib'
-$:.unshift '.'
-
-
-# On my laptop under normal load (e.g., not very scientific at all)
-# I get the following running in a single thread
-#   ruby 2.3               ~8500 lookups/second
-#   ruby 2.4               ~9100 lookups/second
-#   jruby 9                ~20k  lookups/second
-#   jruby 9, old HLB.jar   ~6500 lookups/second
-#   jruby 1.7                    error, can't do named arguments
-#   jruby 1.7, old HLB.jar ~6700 lookups/second
-#
-# The old HLB.jar has a different (worse) algorithm, but is of
-# interest because it's what I'm writing this to replace.
-
-# umich_traject holds .jar files with the old java implementation; see
-# https://github.com/hathitrust/ht_traject/tree/9e8d414fd9bb2c79e243d289c4d39c05d2de27e5/lib/umich_traject
-#
-
-TEST_OLD_STUFF = defined? JRUBY_VERSION and Dir.exist?('./umich_traject')
-if TEST_OLD_STUFF
-  puts "Loading old HLB3.jar stuff"
-  require 'umich_traject/jackson-core-asl-1.4.3.jar'
-  require 'umich_traject/jackson-mapper-asl-1.4.3.jar'
-  require 'umich_traject/apache-solr-umichnormalizers.jar'
-  require 'umich_traject/HLB3.jar'
-  java_import Java::edu.umich.lib.hlb::HLB
-  puts "Initializing HLB"
-  HLB.initialize()
-end
-
-require 'high_level_browse'
-
-h = HighLevelBrowse.load(dir: '.')
-
-cns = File.read('call_numbers.txt').split(/\n/).cycle
-
-puts RUBY_DESCRIPTION
-
-total = 0
-Benchmark.ips do |x|
-  x.config(:time => 25, :warmup => 25)
-
-  x.report("HLB lookups") do
-    total += h[cns.next].count
-  end
-
-  if TEST_OLD_STUFF
-    total = 0
-    x.report("Old java lookups") do
-      total += HLB.categories(cns.next).to_a.count
-    end
-    x.compare!
-  end
-end