store-digest 0.1.3 → 0.3.0

data/lib/store/digest/meta.rb

@@ -1,7 +1,180 @@
-module Store
-  class Digest
-    module Meta
-      # This is an abstract module for metadata operations.
-    end
+# grab the symbols so we don't have to indent
+require_relative 'version'
+
+# This is an abstract module for metadata operations. All required
+# methods are defined, and raise {NotImplementedError}.
+module Store::Digest::Meta
+
+  private
+
+  PRIMARY = :"sha-256"
+
+  DIGESTS = {
+    md5:       16,
+    "sha-1":   20,
+    "sha-256": 32,
+    "sha-384": 48,
+    "sha-512": 64,
+  }.freeze
+
+  INTS   = %i[size ctime mtime ptime dtime flags].map do |k|
+    [k, :to_i]
+  end.to_h.freeze
+
+  protected
+
+  # This method is run on initialization to bootstrap or otherwise
+  # verify the integrity of the database.
+  #
+  # @param options [Hash] whatever the parameters entail; it's your
+  #  driver lol
+  #
+  # @return [void]
+  #
+  def setup **options
+    raise NotImplementedError
+  end
+
+  # Set/add an individual object's metadata to the database.
+  #
+  # @param obj [Store::Digest::Object] the object to store
+  # @param preserve [false, true] flag to preserve modification time
+  #
+  # @return [void]
+  #
+  def set_meta obj, preserve: false
+    raise NotImplementedError
+  end
+
+  # Retrieve a hash of the metadata stored for the given object.
+  #
+  # @param obj [#to_h, URI::ni] something hash-like which has keys that
+  #  correspond to the identifiers for the digest algorithms, or
+  #  otherwise a `ni:` URI.
+  #
+  # @return [Hash]
+  #
+  def get_meta obj, preserve: false
+    raise NotImplementedError
+  end
+
+  # Remove the metadata from the database and return it.
+  #
+  # @see #get_meta
+  #
+  # @param obj [#to_h, URI::ni] The object/identifier
+  #
+  # @return [Hash] The eliminated metadata.
+  #
+  def remove_meta obj, preserve: false
+    raise NotImplementedError
+  end
+
+  # Mark the object's record as "deleted" (but do not actually delete
+  # it) and return the updated record.
+  #
+  # @see #get_meta
+  #
+  # @param obj [#to_h, URI::ni] The object/identifier
+  #
+  # @return [Hash] The updated metadata.
+  #
+  # @return [Hash]
+  #
+  def mark_meta_deleted obj
+    raise NotImplementedError
+  end
+
+  # Retrieve storage statistics from the database itself.
+  #
+  # @return [Hash] global stats for the database.
+  #
+  def meta_get_stats
+    raise NotImplementedError
+  end
+
+  public
+
+  # Wrap the block in a transaction.
+  #
+  # @param block [Proc] whatever you pass into the transaction.
+  #
+  # @return [Object] whatever the block returns.
+  #
+  def transaction &block
+    raise NotImplementedError
+  end
+
+  # Return the set of algorithms initialized in the database.
+  #
+  # @return [Array] the algorithms
+  #
+  def algorithms
+    raise NotImplementedError
+  end
+
+  # Return the primary digest algorithm.
+  #
+  # @return [Symbol] the primary algorithm
+  #
+  def primary
+    raise NotImplementedError
+  end
+
+  # Return the number of objects in the database.
+  #
+  # @return [Integer]
+  #
+  def objects
+    raise NotImplementedError
+  end
+
+  # Return the number of objects whose payloads are deleted but are
+  # still on record.
+  #
+  # @return [Integer]
+  #
+  def deleted
+    raise NotImplementedError
+  end
+
+  # Return the number of bytes stored in the database (notwithstanding
+  # the database itself).
+  #
+  # @return [Integer]
+  #
+  def bytes
+    raise NotImplementedError
+  end
+
+  # Return a list of objects matching the given criteria. The result
+  # set will be the intersection of all supplied parameters. `:type`,
+  # `:charset`, `:encoding`, and `:language` are treated like discrete
+  # sets, while the rest of the parameters are treated like ranges
+  # (two-element arrays). Single values will be coerced into arrays;
+  # single range values will be interpreted as an inclusive lower
+  # bound. To bound only at the top, use a two-element array with its
+  # first value `nil`, like so: `size: [nil, 31337]`. The sorting
+  # criteria are the symbols of the other parameters.
+  #
+  # @param type [nil, String, #to_a]
+  # @param charset [nil, String, #to_a]
+  # @param encoding [nil, String, #to_a]
+  # @param language [nil, String, #to_a]
+  # @param size [nil, Integer, #to_a] byte size range
+  # @param ctime [nil, Time, DateTime, #to_a] creation time range
+  # @param mtime [nil, Time, DateTime, #to_a] modification time range
+  # @param ptime [nil, Time, DateTime, #to_a] medatata property change range
+  # @param dtime [nil, Time, DateTime, #to_a] deletion time range
+  # @param sort [nil, Symbol, #to_a] sorting criteria
+  #
+  # @return [Array] the list
+  #
+  def list type: nil, charset: nil, encoding: nil, language: nil,
+      size: nil, ctime: nil, mtime: nil, ptime: nil, dtime: nil, sort: nil
+    raise NotImplementedError
+  end
+
+  class CorruptStateError < RuntimeError
   end
 end

data/lib/store/digest/object.rb

@@ -3,23 +3,41 @@ require 'store/digest/version'
 require 'uri'
 require 'uri/ni'
 require 'mimemagic'
-require 'mimemagic/overlay'
 
 class MimeMagic
   # XXX erase this when these methods get added
-  unless self.method_defined? :parents
+  unless singleton_class.method_defined? :parents
     def self.parents type
-      TYPES.fetch(type, [nil,[]])[1].map { |t| new t }.uniq
+      TYPES.fetch(type.to_s, [nil,[]])[1].map { |t| new t }.uniq
     end
   end
 
-  unless self.method_defined? :ancestor_types
-    def self.ancestor_types type
-      parents(type).map { |t| ancestors(t) }.flatten.uniq
+  unless method_defined? :parents
+    def parents
+      out = TYPES.fetch(type.to_s.downcase, [nil, []])[1].map do |x|
+        self.class.new x
+      end
+      # add this unless we're it
+      out << self.class.new('application/octet-stream') if
+        out.empty? and type.downcase != 'application/octet-stream'
+
+      out.uniq
+    end
+  end
+
+  unless method_defined? :lineage
+    def lineage
+      ([self] + parents.map { |t| t.lineage }.flatten).uniq
     end
   end
 
-  unless self.method_defined? :binary?
+  unless method_defined? :descendant_of?
+    def descendant_of? type
+      lineage.map(&:type).include? type.to_s.downcase
+    end
+  end
+
+  unless singleton_class.method_defined? :binary?
     def self.binary? thing
       sample = nil
 
@@ -42,15 +60,80 @@ class MimeMagic
     end
   end
 
-  unless self.method_defined? :default_type
+  unless singleton_class.method_defined? :default_type
     def self.default_type thing
       new self.binary?(thing) ? 'application/octet-stream' : 'text/plain'
     end
   end
 end
 
+# Store entry object class.
+#
 class Store::Digest::Object
 
+  # These is a struct for the bank of flags, with a couple of extra
+  # methods for parsing
+  #
+  Flags = Struct.new(
+    'Flags',
+    :type_checked, :type_valid, :charset_checked, :charset_valid,
+    :encoding_checked, :encoding_valid, :syntax_checked, :syntax_valid, :cache
+  ) do |name|
+
+    # Initialize a struct of flags from arbitrary input
+    #
+    # @param arg [Store::Digest::Object::Flags, Integer, #to_h, #to_a]
+    #
+    # @return [Store::Digest::Object::Flags]
+    #
+    def self.from arg
+      # get the length since we use it in a few places
+      len = self.members.size
+
+      if arg.is_a? Integer
+        tmp = arg.digits(2).first(len)
+      elsif arg.is_a? self
+        # noop
+        return arg
+      elsif arg.is_a? Hash
+        tmp = arg.slice(*self.members).transform_values do |v|
+          !!(v && v != 0)
+        end
+        return self.[](**tmp)
+      elsif arg.respond_to? :to_a
+        tmp = arg.to_a.first(len)
+      else
+        raise ArgumentError, 'Input must be an integer or array'
+      end
+
+      # append these
+      tmp += [false] * (len - tmp.size) if tmp.size < len
+
+      # make sure these are true/false
+      tmp.map! { |b| !!(b && b != 0) }
+
+      # we do this because `new` doesn't do this
+      self.[](*tmp)
+    end
+
+    # Turn an arbitrary {Array} back into an {Integer}.
+    #
+    # @param array [Array]
+    #
+    # @return [Integer]
+    #
+    def self.to_i array
+      array.to_a.reverse.reduce(0) { |acc, b| (acc << 1) | (b ? 1 : 0) }
+    end
+
+    # wish there was a cleaner way to do derive individual instance
+    # methods from class methods
+    begin
+      cm = self.method :to_i
+      define_method(:to_i) { cm.call self.to_a }
+    end
+  end
+
   private
 
   SAMPLE    = 2**13 # must be big enough to detect ooxml
@@ -88,13 +171,14 @@ class Store::Digest::Object
   ENCODING_VALID   = 1 << 5
   SYNTAX_CHECKED   = 1 << 6
   SYNTAX_VALID     = 1 << 7
+  IS_CACHE         = 1 << 8
 
   LABELS = {
     size:     'Size (Bytes)',
     ctime:    'Added to Store',
     mtime:    'Last Modified',
     ptime:    'Properties Modified',
-    dtime:    'Deleted',
+    dtime:    'Deleted (Expires)',
     type:     'Content Type',
     language: '(Natural) Language',
     charset:  'Character Set',
@@ -132,7 +216,7 @@ class Store::Digest::Object
 
   # Create a new object, naively recording whatever is handed
   #
-  # @note use {.scan} or {#scan} to populate 
+  # @note use {.scan} or {#scan} to populate
   #
   # @param content [IO, String, Proc, File, Pathname, ...] some content
   # @param digests [Hash] the digests ascribed to the content
@@ -148,7 +232,9 @@ class Store::Digest::Object
   # @param flags [Integer] validation state flags
   # @param strict [true, false] raise an error on bad input
   # @param fresh [true, false] assert "freshness" of object vis-a-vis the store
+  #
   # @return [Store::Digest::Object] the object in question
+  #
   def initialize content = nil, digests: {}, size: 0,
       type: 'application/octet-stream', charset: nil, language: nil,
       encoding: nil, ctime: nil, mtime: nil, ptime: nil, dtime: nil,
@@ -211,19 +297,17 @@ class Store::Digest::Object
       instance_variable_set "@#{k}", v
     end
 
-    # size and flags should be non-negative integers
-    %i[size flags].each do |k|
-      x = b.local_variable_get k
-      v = case x
-          when nil then 0
-          when Integer
-            raise ArgumentError, "#{k} must be non-negative" if x < 0
-            x
-          else
-            raise ArgumentError, "#{k} must be nil or an Integer"
-          end
-      instance_variable_set "@#{k}", v
-    end
+    # set the flags
+    @flags = Flags.from(flags || 0)
+
+    @size = case size
+            when nil then 0
+            when Numeric
+              raise ArgumentError, 'size must be non-negative' if size < 0
+              size.to_i
+            else
+              raise ArgumentError, 'size must be nil or Numeric'
+            end
 
     # the following can be strings or symbols:
     TOKENS.keys.each do |k|
@@ -274,7 +358,7 @@ class Store::Digest::Object
 
     # sane default for mtime
     @mtime = coerce_time(mtime || @mtime ||
-      (content.respond_to?(:mtime) ? content.mtime : Time.now), :mtime)
+      (content.respond_to?(:mtime) ? content.mtime : Time.now(in: ?Z)), :mtime)
 
     # eh, *some* code reuse
     b = binding
@@ -321,15 +405,21 @@ class Store::Digest::Object
       [k, URI::NI.compute(v, algorithm: k).freeze]
     end.to_h.freeze
 
+    # ensure there is the most generic of possible types
+    type ||= 'application/octet-stream'.freeze
+
     # obtain the sampled content type
     ts = MimeMagic.by_magic(sample) || MimeMagic.default_type(sample)
     if content.respond_to? :path
       # may as well use the path if it's available and more specific
-      ps = MimeMagic.by_path(content.path)
+      ps = MimeMagic.by_path(content.path.to_s)
       # XXX the need to do ts.to_s is a bug in mimemagic
-      ts = ps if ps and ps.child_of?(ts.to_s)
+      ts = ps if ps and ps.descendant_of?(ts.to_s)
     end
-    @type = !type || ts.child_of?(type) ? ts.to_s : type
+
+    # set the type to ts if it is more specific
+    @type = ts.descendant_of?(type.to_s) ? ts.to_s.freeze :
+      type.to_s.dup.downcase.freeze
 
     self
   end
@@ -337,9 +427,14 @@ class Store::Digest::Object
   # Determine (or set) whether the object is "fresh", i.e. whether it
   # is new (or restored), or had been previously been in the store.
   #
-  # @param state [true, false]
-  def fresh? state = nil
-    state.nil? ? @fresh : @fresh = !!state
+  # @return [true, false]
+  #
+  def fresh?
+    !!@fresh
+  end
+  
+  def fresh= state
+    @fresh = !!state
   end
 
   # Return the algorithms used in the object.
@@ -385,53 +480,63 @@ class Store::Digest::Object
     !@digests.empty?
   end
 
+  # Returns whether the object is cache.
+  #
+  # @return [false, true]
+  #
+  def cache?
+    !!@flags.cache
+  end
+
+  # XXX i'm keeping these as-is for now
+
   # Returns true if the content type has been checked.
   # @return [false, true]
   def type_checked?
-    0 != @flags & TYPE_CHECKED
+    0 != @flags.to_i & TYPE_CHECKED
   end
 
   # Returns true if the content type has been checked _and_ is valid.
   # @return [false, true]
   def type_valid?
-    0 != @flags & (TYPE_CHECKED|TYPE_VALID)
+    0 != @flags.to_i & (TYPE_CHECKED|TYPE_VALID)
   end
 
   # Returns true if the character set has been checked.
   # @return [false, true]
   def charset_checked?
-    0 != @flags & CHARSET_CHECKED
+    0 != @flags.to_i & CHARSET_CHECKED
   end
 
   # Returns true if the character set has been checked _and_ is valid.
   # @return [false, true]
   def charset_valid?
-    0 != @flags & (CHARSET_CHECKED|CHARSET_VALID)
+    0 != @flags.to_i & (CHARSET_CHECKED|CHARSET_VALID)
   end
 
   # Returns true if the content encoding (e.g. gzip, deflate) has
   # been checked.
   # @return [false, true]
   def encoding_checked?
-    0 != @flags & ENCODING_CHECKED
+    0 != @flags.to_i & ENCODING_CHECKED
   end
 
   # Returns true if the content encoding has been checked _and_ is valid.
   # @return [false, true]
   def encoding_valid?
-    0 != @flags & (ENCODING_CHECKED|ENCODING_VALID)
+    0 != @flags.to_i & (ENCODING_CHECKED|ENCODING_VALID)
   end
 
   # Returns true if the blob's syntax has been checked.
   # @return [false, true]
   def syntax_checked?
-    0 != @flags & SYNTAX_CHECKED
+    0 != @flags.to_i & SYNTAX_CHECKED
   end
 
   # Returns true if the blob's syntax has been checked _and_ is valid.
   # @return [false, true]
   def syntax_valid?
-    0 != @flags & (SYNTAX_CHECKED|SYNTAX_VALID)
+    0 != @flags.to_i & (SYNTAX_CHECKED|SYNTAX_VALID)
   end
 
   %i[ctime mtime ptime dtime].each do |k|
@@ -488,7 +593,7 @@ class Store::Digest::Object
     # now the validation statuses
     out << "Validation:\n"
     FLAG.each_index do |i|
-      x = flags >> (3 - i) & 3
+      x = flags.to_i >> (3 - i) & 3
       out << ("  %-16s: %s\n" % [FLAG[i], STATE[x]])
     end
 

data/lib/store/digest/version.rb

@@ -1,5 +1,5 @@
 module Store
   class Digest
-    VERSION = "0.1.3"
+    VERSION = "0.3.0"
   end
 end

data/lib/store/digest.rb

@@ -67,7 +67,7 @@ class Store::Digest
 
     extend driver
 
-    # 
+    #
     setup(**options)
   end
 
@@ -79,8 +79,8 @@ class Store::Digest
 
   # alias_method :inspect, :to_s
 
-  # Add an object to the store. Takes pretty much anything that
-  # 
+  # Add an object to the store. Takes pretty much anything that makes
+  # sense to throw at it.
   #
   # @note Prefabricated {Store::Digest::Object} instances will be
   #   rescanned.
@@ -96,15 +96,19 @@ class Store::Digest
   # @param mtime [Time] the modification time, if not "now"
   # @param strict [true, false] strict checking on metadata input
   # @param preserve [false, true] preserve existing modification time
+  #
   # @return [Store::Digest::Object] The (potentially pre-existing) entry
+  #
   def add obj, type: nil, charset: nil, language: nil, encoding: nil,
       mtime: nil, strict: true, preserve: false
     return unless obj
-    #transaction do # |txn|
+
+    transaction do # |txn|
       obj = coerce_object obj, type: type, charset: charset,
         language: language, encoding: encoding, mtime: mtime, strict: strict
       raise ArgumentError, 'We need something to store!' unless obj.content?
 
+      # this method is helicoptered in
       tmp = temp_blob
 
       # XXX this is stupid; figure out a better way to do this
@@ -121,6 +125,7 @@ class Store::Digest
 
       # set_meta will return nil if there is no difference in what is set
       if h = set_meta(obj, preserve: preserve)
+        # warn h.inspect
         # replace the object
 
         content = obj.content
@@ -135,30 +140,34 @@ class Store::Digest
 
         # now settle the blob into storage
         settle_blob obj[primary].digest, tmp, mtime: obj.mtime
-        #txn.commit
       else
         tmp.close
         tmp.unlink
 
+        # warn "got here lolol"
+
         # eh just do this
         obj = get obj
-        obj.fresh? false # object is not fresh since we already have it
+        obj.fresh = false # object is not fresh since we already have it
       end
 
       obj
-    #end
+    end
   end
 
   # Retrieve an object from the store.
-  # @param 
+  #
+  # @param obj [URI, Store::Digest::Object]
+  #
+  # @return [Store::Digest::Object, nil]
   def get obj
-    body = -> do
+    transaction readonly: true do
       obj = coerce_object obj
-      h = get_meta(obj) or return # bail if this does not exist
-      b = get_blob h[:digests][primary].digest # may be nil
-      Store::Digest::Object.new b, **h
+      if h = get_meta(obj) # bail if this does not exist
+        b = get_blob h[:digests][primary].digest # may be nil
+        Store::Digest::Object.new b, **h
+      end
     end
-    transaction(&body)
   end
 
   # Remove an object from the store, optionally "forgetting" it ever existed.
@@ -171,23 +180,19 @@ class Store::Digest
       obj.scan digests: algorithms, blocksize: 2**20
     end
 
-    # remove blob and mark metadata entry as deleted
-    meta = nil
+    # remove or mark metadata entry as deleted and remove blob
     transaction do
-      meta = forget ? remove_meta(obj) : mark_meta_deleted(obj)
-    end
-
-    if meta
-      if blob = remove_blob(meta[:digests][primary].digest)
-        return Store::Digest::Object.new blob, **meta
+      if meta = forget ? remove_meta(obj) : mark_meta_deleted(obj)
+        if blob = remove_blob(meta[:digests][primary].digest)
+          Store::Digest::Object.new blob, **meta
+        end
       end
     end
-    nil
   end
 
   # Remove an object from the store and "forget" it ever existed,
   # i.e., purge it from the metadata.
-  # 
+  #
   def forget obj
     remove obj, forget: true
   end

data/store-digest.gemspec

@@ -11,12 +11,12 @@ Gem::Specification.new do |spec|
   spec.license       = 'Apache-2.0'
   spec.homepage      = 'https://github.com/doriantaylor/rb-store-digest'
   spec.summary       = 'Lightweight, multi-digest content-addressable store'
-  spec.description   = <<-DESC
+  spec.description   = <<~DESC
   DESC
 
   spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
     `git ls-files -z`.split("\x0").reject do |f|
-      f.match(%r{^(test|spec|features)/}) 
+      f.match(%r{^(test|spec|features)/})
     end
   end
   spec.bindir        = 'exe'
@@ -24,16 +24,18 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   # ruby
-  spec.required_ruby_version = '~> 2.0'
+  spec.required_ruby_version = '>= 3.0'
 
   # dev/test dependencies
-  spec.add_development_dependency 'bundler', '~> 2.1'
-  spec.add_development_dependency 'rake',    '~> 13.0'
-  spec.add_development_dependency 'rspec',   '~> 3.9'
+  spec.add_development_dependency 'bundler', '>= 2.1'
+  spec.add_development_dependency 'rake',    '>= 13.0'
+  spec.add_development_dependency 'rspec',   '>= 3.9'
 
   # stuff we use
-  spec.add_runtime_dependency 'base32',    '>= 0.3.2'
-  spec.add_runtime_dependency 'lmdb',      '>= 0.5.3' # my hacks
-  spec.add_runtime_dependency 'mimemagic', '>= 0.3.3'
+  spec.add_runtime_dependency 'base64',    '~> 0.3' # stop it complaining
+  spec.add_runtime_dependency 'base32',    '~> 0.3', '>= 0.3.2'
+  spec.add_runtime_dependency 'lmdb',      '~> 0.7', '>= 0.7.1' # my hacks
+#  spec.add_runtime_dependency 'mimemagic', '>= 0.4.3', '< 0.5'
+  spec.add_runtime_dependency 'mimemagic', '>= 0.4.3'
   spec.add_runtime_dependency 'uri-ni',    '>= 0.1.4'
 end