sgfa 0.1.0

data/lib/sgfa/lock_fs.rb

@@ -0,0 +1,141 @@
+#
+# Simple Group of Filing Applications
+# Locking using filesystem locks
+#
+# Copyright (C) 2015 by Graham A. Field.
+#
+# See LICENSE.txt for licensing information.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+require_relative 'error'
+
+module Sgfa
+
+
+##########################################################################
+# Lock based on file locks
+class LockFs
+
+
+  ################################
+  # Lock state
+  #
+  # @return [Symbol] :closed, :unlocked, :shared, or :exclusive
+  attr_reader :state
+
+
+  ################################
+  # Initialize
+  def initialize
+    @file = nil
+    @state = :closed
+  end # def initialize
+
+
+  ################################
+  # Open lock file
+  #
+  # @todo Handle exceptions from File.open()
+  #
+  # @param fnam [String] File name of lock file
+  # @return [String] Contents of the lock file
+  # @raise [Error::Sanity] if lock already open
+  def open(fnam)
+    raise Error::Sanity, 'Lock file already open' if @file
+    @file = File.open(fnam, 'r', :encoding => 'utf-8')
+    @state = :unlocked
+    return @file.read
+  end # def open()
+
+
+  ################################
+  # Close lock file
+  #
+  # @raise [Error::Sanity] if lock not open
+  def close
+    raise Error::Sanity, 'Lock file not open' if !@file
+    @file.close
+    @file = nil
+    @state = :closed
+  end # def close
+
+
+  ################################
+  # Take exclusive lock
+  #
+  # @raise [Error::Sanity] if lock not open
+  def exclusive
+    raise Error::Sanity, 'Lock file not open' if !@file
+
+    if @state == :exclusive
+      return
+    elsif @state == :shared
+      @file.flock(File::LOCK_UN)
+    end
+
+    @file.flock(File::LOCK_EX)
+    @state = :exclusive
+  end # def exclusive
+
+
+  ################################
+  # Take shared lock
+  #
+  # @raise [Error::Sanity] if lock not open
+  def shared
+    raise Error::Sanity, 'Lock file not open' if !@file
+    return if @state == :shared
+    @file.flock(File::LOCK_SH)
+    @state = :shared
+  end # def shared
+
+
+  ################################
+  # Release lock
+  #
+  # @raise [Error::Sanity] if lock not open
+  def unlock
+    raise Error::Sanity, 'Lock file not open' if !@file
+    return if @state == :unlocked
+    @file.flock(File::LOCK_UN)
+    @state = :unlocked
+  end # def unlock
+
+
+  ################################
+  # Run block while holding exclusive lock
+  #
+  # @param rel [Boolean] Release lock on return
+  # @raise (see #exclusive)
+  def do_ex(rel=true)
+    exclusive
+    begin
+      ret = yield
+    ensure
+      unlock if rel
+    end
+    return ret
+  end # def do_ex
+
+
+  ################################
+  # Run block while holding shared lock
+  #
+  # @param rel [Boolean] Release lock on return
+  # @riase (see #shared)
+  def do_sh(rel=true)
+    shared
+    begin
+      ret = yield
+    ensure
+      unlock if rel
+    end
+    return ret
+  end # def do_sh
+
+
+end # class LockFs
+
+end # module Sgfa

data/lib/sgfa/state_fs.rb

@@ -0,0 +1,342 @@
+#
+# Simple Group of Filing Applications
+# Jacket state using filesystem storage.
+#
+# Copyright (C) 2015 by Graham A. Field.
+#
+# See LICENSE.txt for licensing information.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+require 'fileutils'
+require 'tempfile'
+
+require_relative 'error'
+
+module Sgfa
+
+
+#####################################################################
+# Maintains {Jacket} state using filesystem storage.
+#
+# This stores the current {History} number for a {Jacket} as well as
+# the current revision numbers for each {Entry}.  It also maintains
+# for each tag, a date/time ordered list of all {Entry} items where
+# the current revision contains that tag.  Finally, it keeps a list of
+# all tags which are present in the {Jacket}.
+#
+# The current revision of each entry is stored in a file named
+# "_state" with each line consisting of a 9-digit current
+# revision (zero padded) followed by newline.  The current {History}
+# number is stored on line zero, with each {Entry} on the corresponding
+# line.
+#
+# The list of tags is in a file named "_list" and consists of
+# each tag followed by a newline.
+#
+# Each tag is kept in a seperate file, named with the tag name.  Each
+# line consists of a UTC date/time in "YYYY-MM-DD HH:MM:SS" format, 
+# followed by a space, and the 9-digit (zero padded) {Entry} number,
+# with a newline.  A "_all" tag tracks all {Entry}s.
+#
+class StateFs
+
+
+  # Tag which has all {Entry} items in a {Jacket}
+  TagAll = '_all'
+
+  # File name to store list of all tags
+  TagList = '_list'
+
+  # File name used to store the current {History} and {Entry} revision
+  # numbers
+  TagState = '_state'
+
+  # Size (bytes) of each entry revision
+  EntrySize = 10
+
+  # Size (bytes) of each tag date/time and entry listing
+  TagSize = 30
+
+  # Regular expression to get date/time string and entry number
+  # from a tag file
+  TagReg = /^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) (\d{9})$/
+
+  private_constant :TagAll, :TagList, :TagState, :EntrySize, :TagSize,
+    :TagReg
+
+
+  #####################################
+  # Create a new state
+  #
+  # @param path [String] Path to the directory containing the state
+  # @raise [Error::Conflict] if path already exists
+  def self.create(path)
+    begin
+      Dir.mkdir(path)
+    rescue Errno::EEXIST
+      raise Error::Conflict, 'State path already exists'
+    end
+    FileUtils.touch(File.join(path, TagList))
+    File.open(File.join(path, TagState), 'wb'){|fi| fi.write "000000000\n"}
+  end # def self.create()
+
+
+  #####################################
+  # Initialize new state, optionally opening
+  #
+  # @param path [String] Path to the directory containing the state
+  # @raise (see #open)
+  def initialize(path=nil)
+    open(path) if path
+  end # def initialize()
+
+
+  #####################################
+  # Open the state
+  #
+  # @param path [String] Path to the directory containing the state
+  # @raise [Error::NonExistent] if path does not exist
+  # @return [StateFs] self
+  def open(path)
+    fn = File.join(path, TagState)
+    begin
+      @file = File.open(fn, 'r+b')
+    rescue Errno::ENOENT
+      raise Error::NonExistent, 'State path does not exist'
+    end
+    @path = path.dup
+    return self
+  end # def open()
+
+
+  #####################################
+  # Close the state
+  #
+  # @raise [Error::Sanity] if state is not open
+  # @return [StateFs] self
+  def close
+    raise Error::Sanity, 'State not open' if !@path
+    @file.close
+    @path = nil
+    return self
+  end # def close
+
+
+  #####################################
+  # Reset to empty state
+  #
+  # @raise [Error::Sanity] if state is not open
+  # @return (see #open)
+  def reset
+    raise Error::Sanity, 'State not open' if !@path
+
+    path = @path
+    self.close
+
+    FileUtils.remove_dir(path)
+    Dir.mkdir(path)
+    FileUtils.touch(File.join(path, TagList))
+    File.open(File.join(path, TagState), 'wb'){|fi| fi.write "000000000\n"}
+    
+    return open(path)
+  end # def reset
+
+
+  #####################################
+  # Set current revision number for an entry
+  #
+  # @param enum [Integer] Entry number
+  # @param rnum [Integer] Current revision number
+  # @raise [Error::Sanity] if state is not open
+  # @raise [ArgumentError] if enum or rnum is negative
+  # @return [StateFs] self
+  def set(enum, rnum)
+    raise Error::Sanity, 'State not open' if !@path
+    if enum < 0 || rnum < 0
+      raise ArgumentError, 'Invalid entry/revision numbers'
+    end
+
+    @file.seek(enum*EntrySize, IO::SEEK_SET)
+    @file.write("%09d\n" % rnum)
+    return self
+  end # def set
+
+
+  #####################################
+  # Get current revision number for an entry
+  #
+  # @param enum [Integer] Entry number
+  # @return [Integer] Current revision number
+  # @raise [Error::Sanity] if state not open
+  # @raise [ArgumentError] if enum is negative
+  # @raise [Error::NonExistent] if entry number does not exist
+  def get(enum)
+    raise Error::Sanity, 'State not open' if !@path
+    raise ArgumentError, 'Invalid entry number' if enum < 0
+    
+    @file.seek(enum*EntrySize, IO::SEEK_SET)
+    res = @file.read(EntrySize)
+    raise Error::NonExistent, 'Entry does not exist' if !res || res[0] == "\x00"
+    return res.to_i
+  end # def get()
+
+
+  #####################################
+  # Read list of tags
+  #
+  # @return [Array] List of strings containing tag names
+  # @raise [Error::Sanity] if state not open
+  # @raise [Error::Corrupt] if tag list is missing
+  def list
+    raise Error::Sanity, 'State not open' if !@path
+
+    ftn = File.join(@path, TagList)
+    begin
+      txt = File.read(ftn)
+    rescue Errno::ENOENT
+      raise Error::Corrupt, 'Unable to read tag list'
+    end
+    return txt.lines.map{|tg| tg.chomp }
+  end # def list
+
+
+  #####################################
+  # Read entry numbers from a tag
+  #
+  # @param name [String] Tag name
+  # @param offs [Integer] Offset to begin reading
+  # @param max [Integer] Maximum number of entries to return
+  # @raise [Error::Sanity] if state not open
+  # @raise [Error::NonExistent] if tag does not exist
+  # @raise [Error::Corrupt] if file format is bad
+  # @return [Array] Total number of entries in the tag, and possibly empty
+  #   array of entry numbers.
+  def tag(name, offs, max)
+    raise Error::Sanity, 'State not open' if !@path
+
+    fn = File.join(@path, name)
+    begin
+      fi = File.open(fn, 'rb')
+    rescue Errno::ENOENT
+      raise Error::NonExistent, 'Tag does not exist'
+    end
+
+    ents = []
+    size = nil
+    begin
+      size = fi.size / TagSize
+      return [size, ents] if( offs > size || max == 0 )
+      num = (offs+max > size) ? (size - offs) : max
+      fi.seek(((size-offs-num)*TagSize), IO::SEEK_SET)
+      num.times do
+        ln = fi.read(TagSize)
+        ma = TagReg.match(ln)
+        raise Error::Corrupt, 'Bad tag format' unless ma
+        ents.push( ma[2].to_i )
+      end
+    ensure
+      fi.close
+    end
+
+    return [size, ents]
+  end # def tag()
+
+
+  #####################################
+  # Update tags based on new {History}
+  #
+  # @param cng [Hash] Changes in the format returned by {History#next}
+  # @raise [Error::Sanity] if state not open
+  # @raise [Error::Corrupt] if file format is bad
+  # @raise [Error::Corrupt] if list of tags is missing
+  # @raise [Error::Corrupt] if existing tag is missing
+  # @return [StateFs] self
+  def update(cng)
+
+    # read list of tags
+    changed = false
+    thash = {}
+    self.list.each{|tag| thash[tag] = true }
+
+    cng.each do |tag, hc|
+      cnt = 0
+
+      # sorted changes
+      se = hc.to_a.select{|en, ti| ti }.sort{|aa, bb| aa[1] <=> bb[1] }
+
+      # files
+      fn = File.join(@path, tag)
+      if thash[tag]
+        begin
+          oldf = File.open(fn, 'rb')
+        rescue Errno::ENOENT
+          raise Error::Corrupt, 'Existing tag is missing'
+        end
+      else
+        oldf = nil
+      end
+      newf = Tempfile.new('state', @path, :encoding => 'ASCII-8BIT')
+
+      # merge new into old file
+      while oldf && ln = oldf.read(TagSize)
+        unless ma = TagReg.match(ln)
+          newf.close!
+          raise Error::Corrupt, 'Bad tag format'
+        end
+
+        tme = ma[1]
+        enum = ma[2].to_i
+
+        next if hc.has_key?(enum)
+
+        while se.size > 0 && tme > se[0][1]
+          ne, nt = se.shift
+          newf.write("%s %09d\n" % [nt, ne])
+          cnt += 1
+        end
+
+        newf.write(ln)
+        cnt += 1
+      end
+      oldf.close() if oldf
+
+      # write out any remaining new
+      while ary = se.shift
+        ne, nt = ary
+        newf.write("%s %09d\n" % [nt, ne])
+        cnt += 1
+      end
+
+      # adjust files
+      if cnt == 0
+        if oldf
+          File.unlink(fn)
+          thash.delete(tag)
+          changed = true
+        end
+      else
+        FileUtils.ln(newf.path, fn, :force => true)
+        if !oldf
+          thash[tag] = true
+          changed = true
+        end
+      end
+      newf.close!
+    end
+
+    # write list of tags
+    if changed
+      fnl = File.join(@path, TagList)
+      File.open(fnl, 'w', :encoding => 'utf-8') do |fi|
+        thash.each_key{|tag| fi.puts tag }
+      end
+    end
+
+    return self
+  end # def update()
+
+end # class StateFs
+
+end # module Sgfa

data/lib/sgfa/store_fs.rb

@@ -0,0 +1,214 @@
+#
+# Simple Group of Filing Applications
+# Jacket store using filesystem storage.
+#
+# Copyright (C) 2015 by Graham A. Field.
+#
+# See LICENSE.txt for licensing information.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the
+# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+require 'tempfile'
+require 'fileutils'
+
+require_relative 'error'
+
+module Sgfa
+
+
+#####################################################################
+# Stores copies of {History}, {Entry}, and attached files that are
+# in a {Jacket} using filesystem storage.
+#
+# Storage and retrieval are based on type (i.e. History, Entry, or
+# attached file) and the item hash.  The item hash is generated from
+# the {Jacket} id_hash, the type, and the specific number.  This allows
+# for a single store to serve as the repository of multiple {Jacket}s
+# if desired (e.g. using a cloud permanent object store).
+#
+# Each item is stored in a file, organized into directories based on the
+# first two characters of the item hash, a file name consisting of the
+# remaining item hash, dash, and a character indicating the type.
+#
+class StoreFs
+
+
+  #####################################
+  # Initialize a new store object, optionally opening
+  #
+  # @param (see #open)
+  # @raise (see #open)
+  def initialize(path=nil)
+    open(path) if path
+  end # def initialize()
+
+
+  #####################################
+  # Create a new store
+  # @param path [String] Path to the directory containing the store
+  # @raise [Error::Conflict] if path already exists
+  def self.create(path)
+    begin
+      Dir.mkdir(path)
+    rescue Errno::EEXIST
+      raise Error::Conflict, 'Store already exists'
+    end
+  end # def self.create()
+
+
+  #####################################
+  # Open the store
+  #
+  # @param path [String] Path to the directory containing the store
+  # @raise [Error::NonExistent] if path does not exist
+  # @return [StoreFs] self
+  def open(path)
+    raise Error::NonExistent, 'Store does not exist' if !File.directory?(path)
+    @path = path
+    return self
+  end # open()
+
+
+  #####################################
+  # Close the store
+  #
+  # @raise [Error::Sanity] if store is not open
+  # @return [StoreFs] self
+  def close
+    raise Error::Sanity, 'Store is not open' if !@path
+    @path = nil
+    return self
+  end # def close
+
+
+  #####################################
+  # File name for an item
+  #
+  # @param type [Symbol] Type of the item
+  # @param item [String] Item hash identifier
+  # @raise [NotImplementedError] if type is not valid
+  # @return [String] the file name
+  def _fn(type, item)
+    case type
+      when :entry then ext = 'e'
+      when :history then ext = 'h'
+      when :file then ext = 'f'
+      else raise NotImplementedError, 'Invalid item type'
+    end
+
+    return '%s/%s/%s-%s' % [@path, item[0,2], item[2..-1], ext]
+  end # def _fn()
+  private :_fn
+
+
+  #####################################
+  # Get a temp file to use to create an item for storage
+  #
+  # @note Encoding on the file is set to 'utf-8' by default
+  # @raise [Error::Sanity] if store not open
+  # @return [Tempfile] the temporary file
+  def temp
+    raise Error::Sanity, 'Store not open' if !@path
+    Tempfile.new('blob', @path, :encoding => 'utf-8')
+  end # def temp
+
+
+  #####################################
+  # Read an item
+  #
+  # @param (see #_fn)
+  # @raise [NotImplementedError] if type is not valid
+  # @raise [Error::Sanity] if store not open
+  # @return [File] Item opened in read only format or false if not found
+  def read(type, item)
+    raise Error::Sanity, 'Store not open' if !@path
+
+    fn = _fn(type, item)
+    begin
+      fi = File.open(fn, 'r', :encoding => 'utf-8')
+    rescue Errno::ENOENT
+      return false
+    end
+    return fi
+  end # def read()
+
+
+  #####################################
+  # Write an item
+  #
+  # If content is a string, it will be written to a file and saved.
+  # If content is a file it will be deleted from it's current location.
+  #
+  # @param (see #_fn)
+  # @param cont [File, String] Content to store
+  # @raise [Error::Sanity] if store not open
+  # @raise [NotImplementedError] if type is not valid
+  def write(type, item, cont)
+    raise Error::Sanity, 'Store not open' if !@path
+    fn = _fn(type, item)
+
+    if cont.is_a?(String)
+      tf = temp
+      tf.write cont
+      cont = tf
+    end
+
+    begin
+      FileUtils.ln(cont.path, fn, :force => true)
+    rescue Errno::ENOENT
+      Dir.mkdir(File.dirname(fn))
+      FileUtils.ln(cont.path, fn)
+    end
+    if cont.respond_to?( :close! )
+      cont.close!
+    else
+      File.unlink(cont, path)
+      cont.close
+    end
+
+    return self
+  end # def write()
+
+
+  #####################################
+  # Delete an item
+  #
+  # @param (see #_fn)
+  # @raise [NotImplementedError] if type is not valid
+  # @raise [Error::Sanity] if store not open
+  # @return [Boolean] if item deleted
+  def delete(type, item)
+    raise Error::Sanity, 'Store not open' if !@path
+    fn = _fn(type, item)
+
+    begin
+      File.unlink(fn)
+    rescue Errno::ENOENT
+      return false
+    end
+    return true
+  end # def delete()
+
+
+  #####################################
+  # Get size of an item in bytes
+  #
+  # @param (see #_fn)
+  # @raise [NotImplementedError] if type is not valid
+  # @raise [Error::Sanity] if store not open
+  # @return [Integer, Boolean] Item size, or false if item does not exist
+  def size(type, item)
+    fn = _fn(type, item)
+    begin
+      size = File.size(fn)
+    rescue Errno::ENOENT
+      return false
+    end
+    return size
+  end # def size()
+
+
+end # class StoreFs
+
+end # module Sgfa