text-checkm 0.7

data/lib/text/checkm.rb

@@ -0,0 +1,2 @@
+require 'text/checkm/manifest'
+require 'text/checkm/entry'

data/lib/text/checkm/checksum.rb

@@ -0,0 +1,40 @@
+require 'digest'
+
+module Text
+  module Checkm
+    class << self
+      # Size (in bytes) to read (in chunks) to compute checksums
+      CHUNK_SIZE = 8 * 1024 * 1024
+
+      # Compute the checksum 'alg' for a file
+      # @param [File] file
+      # @param [String] alg md5, sha1, sha256, dir
+      def checksum(file, alg) # TODO: don't pass file handles around
+        return true unless alg # TODO: something less counterintuitive
+        return File.directory?(file) if alg =~ /dir/
+
+        digest_alg = digest_for(alg)
+        return false unless digest_alg # TODO: something less counterintuitive
+
+        while !file.eof? && (chunk = file.readpartial(CHUNK_SIZE))
+          digest_alg << chunk
+        end
+        digest_alg.hexdigest
+      end
+
+      private
+
+      def digest_for(alg)
+        case alg
+        when /md5/
+          Digest::MD5.new
+        when /sha1/
+          Digest::SHA1.new
+        when /sha256/
+          Digest::SHA2.new(256)
+        end
+      end
+
+    end
+  end
+end

data/lib/text/checkm/entry.rb

@@ -0,0 +1,77 @@
+require 'open-uri'
+require 'time'
+
+require 'text/checkm/checksum'
+
+module Text
+  module Checkm
+    class Entry
+      BASE_FIELDS = %w[sourcefileorurl alg digest length modtime targetfileorurl].freeze
+      attr_reader :values
+
+      def self.create(path, args = {}) # TODO: why is this in this class?
+        base = args[:base] || Dir.pwd
+        alg = args[:alg] || 'md5'
+        file = File.new File.join(base, path)
+
+        format('%s | %s | %s | %s | %s | %s', path, alg, Checkm.checksum(file, alg), File.size(file.path), file.mtime.utc.xmlschema, nil)
+      end
+
+      def initialize(line, manifest = nil)
+        @line = line.strip
+        @include = false
+        @fields = BASE_FIELDS
+        @fields = manifest.fields if manifest && manifest.fields
+        @values = line.split('|').map(&:strip)
+        @manifest = manifest
+      end
+
+      # rubocop:disable Style/MethodMissingSuper
+      def method_missing(sym, *_args)
+        # TODO: something less extreme
+        @values[@fields.index(sym.to_s.downcase) || BASE_FIELDS.index(sym.to_s.downcase)]
+      end
+      # rubocop:enable Style/MethodMissingSuper
+
+      def respond_to_missing?(sym, *)
+        @fields.include?(sym.to_s.downcase) || BASE_FIELDS.include?(sym.to_s.downcase)
+      end
+
+      def valid?
+        source_exists? && valid_checksum? && valid_multilevel? # xxx && valid_length? && valid_modtime?
+      end
+
+      private
+
+      def source
+        file = sourcefileorurl
+        file = file[1..] if file =~ /^@/
+        File.join(@manifest.path, file)
+      end
+
+      def source_exists?
+        File.exist? source
+      end
+
+      def valid_checksum?
+        file = File.new source
+        checksum = Checkm.checksum(file, alg)
+        [true, digest].include?(checksum) # TODO: something less counterintuitive
+      end
+
+      # def valid_length?
+      #   throw NotImplementedError
+      # end
+      #
+      # def valid_modtime?
+      #   throw NotImplementedError
+      # end
+
+      def valid_multilevel?
+        return true unless sourcefileorurl =~ /^@/
+
+        Manifest.parse(URI.open(source).read, path: File.dirname(source))
+      end
+    end
+  end
+end

data/lib/text/checkm/manifest.rb

@@ -0,0 +1,115 @@
+require 'text/checkm/entry'
+
+module Text
+  module Checkm
+    class Manifest
+      def self.parse(str, args = {})
+        Manifest.new str, args
+      end
+
+      attr_reader :version
+      attr_reader :entries
+      attr_reader :fields
+      attr_reader :path
+
+      # rubocop:disable Metrics/MethodLength
+      def initialize(checkm, args = {})
+        @args = args
+        @version = nil
+        @checkm = checkm
+        @lines = checkm.split "\n"
+        @entries = []
+        @eof = false
+        @fields = nil
+        @path = args[:path] # TODO: something less hacky
+        @path ||= Dir.pwd
+
+        parse_lines
+        # xxx error on empty entries?
+        @lines.unshift('#%checkm_0.7') and (@version = '0.7') if @version.nil?
+      end
+
+      # rubocop:enable Metrics/MethodLength
+
+      def valid?
+        return true if @entries.empty?
+
+        @entries.map(&:valid?).none? { |b| b == false }
+      end
+
+      def add(path, args = {})
+        line = Entry.create path, args
+
+        Manifest.new [@lines, line].flatten.join("\n"), @args
+      end
+
+      def remove(path)
+        Manifest.new @lines.reject { |x| x =~ /^@?#{path}/ }.join("\n"), @args
+      end
+
+      def to_s
+        @lines.join("\n")
+      end
+
+      def to_h
+        {}.tap do |h|
+          entries.each do |e|
+            source = e.sourcefileorurl
+            (h[source] ||= []) << e
+          end
+        end
+      end
+
+      private
+
+      # rubocop:disable Metrics/MethodLength
+      def parse_lines
+        @lines.each do |line|
+          case line
+          when /^#%/
+            parse_header line
+          when /^#/
+            parse_comment line
+          when /^$/
+            # do nothing
+          when /^@/
+            parse_line line
+          else
+            parse_line line
+          end
+        end
+      end
+
+      # rubocop:enable Metrics/MethodLength
+
+      # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity
+      def parse_header(line)
+        case line
+        when /^#%checkm/
+          match = /^#%checkm_(\d+)\.(\d+)/.match line
+          @version = "#{match[1]}.#{match[2]}" if match
+        when /^#%eof/
+          @eof = true
+        when /^#%fields/
+          list = line.split('|')
+          list.shift
+          @fields = list.map { |v| v.strip.downcase }
+        when /^#%prefix/
+          # do nothing
+        when /^#%profile/
+          # do nothing
+        end
+      end
+
+      # rubocop:enable Metrics/MethodLength, Metrics/CyclomaticComplexity
+
+      def parse_comment(_line)
+        # do nothing
+      end
+
+      def parse_line(line)
+        @entries << Entry.new(line, self)
+      end
+    end
+  end
+end

data/lib/text/checkm/module_info.rb

@@ -0,0 +1,16 @@
+module Text
+  module Checkm
+    class ModuleInfo
+      NAME = 'text-checkm'.freeze
+      AUTHOR = 'David Moles'.freeze
+      AUTHOR_EMAIL = 'dmoles@berkeley.edu'.freeze
+      SUMMARY = 'Ruby implementation of the Checkm text manifest format'.freeze
+      DESCRIPTION = 'A Ruby implementation of the Checkm checksum-based text manifest format'.freeze
+      LICENSE = 'MIT'.freeze
+      VERSION = '0.7'.freeze
+      HOMEPAGE = 'https://github.com/dmolesUC/checkm'.freeze
+
+      private_class_method :new
+    end
+  end
+end

data/rakelib/bundle.rake

@@ -0,0 +1,8 @@
+namespace :bundle do
+  desc 'Updates the ruby-advisory-db then runs bundle-audit'
+  task :audit do
+    require 'bundler/audit/cli'
+    Bundler::Audit::CLI.start ['update']
+    Bundler::Audit::CLI.start ['check', '--ignore', 'CVE-2015-9284']
+  end
+end

data/rakelib/coverage.rake

@@ -0,0 +1,5 @@
+desc 'Run all specs in spec directory, with coverage'
+task :coverage do
+  ENV['COVERAGE'] ||= 'true'
+  Rake::Task[:spec].invoke
+end

data/rakelib/rubocop.rake

@@ -0,0 +1,16 @@
+require 'rubocop'
+require 'rubocop/rake_task'
+
+desc 'Run rubocop with HTML output'
+RuboCop::RakeTask.new(:rubocop) do |cop|
+  output = ENV['RUBOCOP_OUTPUT'] || 'artifacts/reports/rubocop/index.html'
+
+  cop.formatters = ['html']
+  cop.options = ['--out', output]
+end
+
+desc 'Run RuboCop with auto-correct, and output results to console'
+task :ra do
+  # b/c we want console output, we can't just use `rubocop:auto_correct`
+  RuboCop::CLI.new.run(['--auto-correct'])
+end

data/spec/.rubocop.yml

@@ -0,0 +1,19 @@
+inherit_from: ../.rubocop.yml
+
+Style/ClassAndModuleChildren:
+  Enabled: false
+
+Layout/LineLength:
+  Enabled: false
+
+Metrics/BlockLength:
+  Enabled: false
+
+Metrics/ClassLength:
+  Enabled: false
+
+Metrics/ModuleLength:
+  Enabled: false
+
+Metrics/MethodLength:
+  Enabled: false

data/spec/data/merritt-manifest.checkm

@@ -0,0 +1,9 @@
+#%checkm_0.7
+#%profile | http://uc3.cdlib.org/registry/ingest/manifest/mrt-single-file-batch-manifest
+#%prefix | mrt: | http://merritt.cdlib.org/terms#
+#%prefix | nfo: | http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#
+#%fields | nfo:fileUrl | nfo:hashAlgorithm | nfo:hashValue | nfo:fileSize | nfo:fileLastModified | nfo:fileName | mrt:primaryIdentifier | mrt:localIdentifier | mrt:creator | mrt:title | mrt:date
+http://merritt.cdlib.org/samples/goldenDragon.jpg | md5 | aa59e145dfb2237ecd8cb5ce4f4953ea |  |  | goldenDragon.jpg |  |  | Tracy Seneca | Golden Dragon Restaurant Sign | 2009-09-01T00:00:00
+http://merritt.cdlib.org/samples/tumbleBug.jpg | md5 | 61224ef4f2a8dad552ff57980ed0bf49 |  |  | tumbleBug.jpg |  |  | Tracy Seneca | Tumble Bug, Luna Park.  Sydney
+http://merritt.cdlib.org/samples/generalDrapery.jpg |  |  |  |  | generalDrapery.jpg |  |  |  | General Drapery
+#%eof

data/spec/data/myfirst.checkm

@@ -0,0 +1,5 @@
+#%checkm_0.7
+# My first manifest.  Two files total.
+# Filename        |Algorithm|  Digest
+book/Chapter9.xml |   md5   |  49afbd86a1ca9f34b677a3f09655eae9
+images/r862.png   |   md5   |  408ad21d50cef31da4df6d9ed81b01a7

data/spec/data/test_1/1

@@ -0,0 +1 @@
+1

data/spec/data/two-level-manifest.checkm

@@ -0,0 +1,8 @@
+#%checkm_0.7
+# A two-level manifest.
+
+#Filename|Alg |Checksum                                  |Length
+foo.bar     |sha1|2eacd0da7aa89b094f5121eb2901bf4de2219ef1  | 366
+foo.bar     |md5 |3e83471320227c0797a0c251f28db0c5          | 366
+# This next line "includes" the manifest in file "myfirst".
+@myfirst.checkm    |md5 |1ad6bcd5e4c49420ead2df43406d37fb          | 218

data/spec/lib/text/checkm/checksum_spec.rb

@@ -0,0 +1,24 @@
+require 'spec_helper'
+
+module Text
+  module Checkm
+    describe :checksum do
+      it 'calculates the checksum' do
+        file = 'spec/data/test_1/1'
+        expected = {
+          'md5' => 'b026324c6904b2a9cb4b88d6d61c81d1',
+          'sha1' => 'e5fa44f2b31c1fb553b6021e7360d07d5d91ff5e',
+          'sha256' => '4355a46b19d348dc2f57c046f8ef63d4538ebb936000f3c9ee954a27460dd865'
+        }
+        aggregate_failures 'checksums' do
+          expected.each do |alg, checksum|
+            File.open(file, 'r') do |f|
+              actual = Checkm.checksum(f, alg)
+              expect(actual).to eq(checksum), "Wrong #{alg} checksum for #{file}, expected #{checksum}, was #{actual}"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/spec/lib/text/checkm/entry_spec.rb

@@ -0,0 +1,52 @@
+require 'spec_helper'
+
+module Text
+  module Checkm
+    describe Entry do
+      describe :create do
+        it 'should return a manifest line' do
+          res = Entry.create('LICENSE.md')
+          expect(res).to match(/LICENSE\.md | md5 | a02e647a5dcd1fe38abf74f9f0d44dae | 1149 | \d{4}/)
+        end
+      end
+
+      describe :respond_to_missing? do
+        it 'should return true for all base fields' do
+          entry = Entry.new('book/Chapter9.xml |   md5   |  49afbd86a1ca9f34b677a3f09655eae9')
+          Entry::BASE_FIELDS.each do |f|
+            expect(entry.respond_to?(f.to_sym)).to eq(true)
+          end
+        end
+
+        it 'should return true for all custom fields' do
+          expected_fields = %w[
+            nfo:fileUrl
+            nfo:hashAlgorithm
+            nfo:hashValue
+            nfo:fileSize
+            nfo:fileLastModified
+            nfo:fileName
+            mrt:primaryIdentifier
+            mrt:localIdentifier
+            mrt:creator
+            mrt:title
+            mrt:date
+          ]
+          manifest = Manifest.parse(File.read('spec/data/merritt-manifest.checkm'))
+          entry = manifest.entries[0]
+          expected_fields.each do |f|
+            expect(entry.respond_to?(f.to_sym)).to eq(true)
+          end
+        end
+      end
+
+      describe :valid do
+        it 'handles multi-level manifests' do
+          manifest = Manifest.parse(File.read('spec/data/two-level-manifest.checkm'), path: 'spec/data')
+          entry = manifest.entries[2]
+          expect(entry.valid?).to be_truthy # TODO: something less hacky
+        end
+      end
+    end
+  end
+end

data/spec/lib/text/checkm/manifest_spec.rb

@@ -0,0 +1,157 @@
+require 'spec_helper'
+
+module Text
+  module Checkm
+    describe Manifest do
+      it 'should be valid if empty' do
+        checkm = ''
+        res = Manifest.parse(checkm)
+        expect(res.entries).to be_empty
+        expect(res).to be_valid
+      end
+
+      it 'should ignore comments' do
+        checkm = '#'
+        res = Manifest.parse(checkm)
+        expect(res.entries).to be_empty
+        expect(res).to be_valid
+      end
+
+      it 'should parse the checkm version' do
+        checkm = '#%checkm_0.7'
+        res = Manifest.parse(checkm)
+        expect(res.entries).to be_empty
+        expect(res).to be_valid
+        expect(res.version).to eq('0.7')
+      end
+
+      describe 'simple checkm line' do
+        before(:each) do
+          @checkm = 'book/Chapter9.xml |   md5   |  49afbd86a1ca9f34b677a3f09655eae9'
+          @result = Manifest.parse(@checkm)
+          @line = @result.entries.first
+        end
+
+        it 'should parse one entry' do
+          expect(@result.entries.size).to eq(1)
+        end
+
+        it 'should parse a checkm line' do
+          expect(@line.values[0]).to eq('book/Chapter9.xml')
+          expect(@line.values[1]).to eq('md5')
+          expect(@line.values[2]).to eq('49afbd86a1ca9f34b677a3f09655eae9')
+        end
+
+        it 'should allow name-based lookups' do
+          expect(@line.sourcefileorurl).to eq('book/Chapter9.xml')
+          expect(@line.alg).to eq('md5')
+          expect(@line.digest).to eq('49afbd86a1ca9f34b677a3f09655eae9')
+        end
+      end
+
+      it 'should support custom field names' do
+        checkm = <<~CHECKM
+          #%fields | testa | test b
+          book/Chapter9.xml |   md5   |  49afbd86a1ca9f34b677a3f09655eae9
+        CHECKM
+        checkm.strip!
+
+        res = Manifest.parse(checkm)
+
+        line = res.entries.first
+
+        expect(line.sourcefileorurl).to eq('book/Chapter9.xml')
+        expect(line.testa).to eq('book/Chapter9.xml')
+        expect(line.alg).to eq('md5')
+        expect(line.send(:'test b')).to eq('md5')
+        expect(line.digest).to eq('49afbd86a1ca9f34b677a3f09655eae9')
+      end
+
+      describe 'validity check' do
+        it 'should be valid if the file exists' do
+          checkm = '1 | md5 | b026324c6904b2a9cb4b88d6d61c81d1'
+          res = Manifest.parse(checkm, path: 'spec/data/test_1')
+          expect(res.entries.size).to eq(1)
+          expect(res).to be_valid
+        end
+
+        it 'should be valid if the directory exists' do
+          checkm = 'test_1 | dir'
+          res = Manifest.parse(checkm, path: 'spec/data')
+          expect(res.entries.size).to eq(1)
+          expect(res).to be_valid
+        end
+
+        it 'should be invalid if a file is missing' do
+          checkm = '2 | md5 | b026324c6904b2a9cb4b88d6d61c81d1'
+          res = Manifest.parse(checkm, path: 'spec/data/test_1')
+          expect(res.entries.size).to eq(1)
+          expect(res).not_to be_valid
+        end
+
+        it 'should be invalid if the checksum is different' do
+          checkm = '1 | md5 | zzz'
+          res = Manifest.parse(checkm, path: 'spec/data/test_1')
+          expect(res.entries.size).to eq(1)
+          expect(res).not_to be_valid
+        end
+      end
+
+      describe 'manipulate manifest' do
+        it 'should allow files to be added to an existing manifest' do
+          m = Manifest.parse('')
+          res = m.add('LICENSE.md')
+          expect(res.entries.size).to eq(1)
+          expect(res).to be_valid
+        end
+      end
+
+      it 'should be serializable to a string' do
+        m = Manifest.parse('')
+        n = m.add('LICENSE.md')
+        lines = n.to_s.split "\n"
+        expect(lines[0]).to eq('#%checkm_0.7')
+        expect(lines[1]).to match(/^LICENSE\.md/)
+      end
+
+      it 'parses a two-level manifest' do
+        m = Manifest.parse(File.read('spec/data/two-level-manifest.checkm'))
+        entries = m.entries
+        expect(entries.size).to eq(3)
+        entry = entries[2]
+        expect(entry.sourcefileorurl).to eq('@myfirst.checkm')
+      end
+
+      describe :remove do
+        attr_reader :original
+        attr_reader :modified
+
+        before(:each) do
+          @original = Manifest.parse(File.read('spec/data/two-level-manifest.checkm'))
+          @modified = original.remove('foo.bar')
+        end
+
+        it 'removes entries by name' do
+          expect(modified.entries.size).to eq(1)
+          expect(modified.entries[0].sourcefileorurl).to eq('@myfirst.checkm')
+        end
+
+        it 'does not modify the original' do
+          expect(original.entries.size).to eq(3)
+          expect(original.entries[0].sourcefileorurl).to eq('foo.bar')
+          expect(original.entries[1].sourcefileorurl).to eq('foo.bar')
+        end
+      end
+
+      describe :to_h do
+        it 'returns a hash of entries by source' do
+          manifest = Manifest.parse(File.read('spec/data/two-level-manifest.checkm'))
+          h = manifest.to_h
+          manifest.entries.each do |e|
+            expect(h[e.sourcefileorurl]).to include(e)
+          end
+        end
+      end
+    end
+  end
+end