file_indexer 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b0df350bb38bc48552894171e4ed305c67f152cf4bcac83f46af44911ff659f0
-  data.tar.gz: 59d9c6aa03fc9fe4e6bc4311c6a5295384bff30410cfe17e4b847c7d13618f02
+  metadata.gz: 7fc83ca7cc27629d1d6c8acbbc0e581158c4e960c5f0ab98c038eeff45118527
+  data.tar.gz: a1f07b862588d490953a6b9244fdc28e74caf821e8775abe028ccad20e991ff7
 SHA512:
-  metadata.gz: a088e4f5918659b8f250ea09137a2b781b2b0978701b39783835c5afce08ea70586ab81f45af4e787e554c4b217e2ba26e5930d4a7db2b9e8aea62b0b186ac81
-  data.tar.gz: e806cec507fb7a9b8b8687e44671a03ae67c51e09c2398e9ce79de2b93ccc88096307a7d3f40556fe2eef3712c466c434f719c2e6c65501b4c1531010078031d
+  metadata.gz: 753d11d8e8f3c96e0c496e21c7d5eab05b36e7cc55040e6480ccb16bde50119b5d34c99881cc8cc192c0e6c54ece347510758b87fe1f443b38909f9848aca564
+  data.tar.gz: ef2afddf154e51691352eff1943af6fa82b9876dd34a6359d12c66da677c853336a037c715cfea50e12c124cd0bfb471912c1268605afa8ba8c02246852765a8

data/lib/file_indexer.rb

@@ -2,7 +2,7 @@ require_relative 'file_indexer/indexer'
 
 # Namespace for a simple file indexer
 module FileIndexer
-  VERSION = [1, 0, 0]
+  VERSION = [1, 0, 1]
 
   # Equivalent to FileIndexer::Indexer.new(dirs, exts, db_file, &action)
   def self.new(dirs, exts, db_file, &action)

data/lib/file_indexer/indexer.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 require 'listen'
 require 'observer'
+require 'pathname'
 
 module FileIndexer
 
@@ -22,7 +23,7 @@ module FileIndexer
     attr_reader :db_file
 
     def initialize(dirs, exts, db_file, &action)
-      @dirs    = dirs
+      @dirs    = dirs.map { |d| sanitize_dirpath(d) }
       @exts    = exts
       @db_file = db_file
       @action  = action
@@ -31,6 +32,7 @@ module FileIndexer
       @listener = nil
 
       @files.merge!(read_db)
+      prune!
       index_all!
     end
 
@@ -41,25 +43,31 @@ module FileIndexer
 
     # @see #index_all
     def index_all!(dirs = @dirs, exts = @exts)
-      @files = index_all(dirs, exts)
+      files.merge!(index_all(dirs, exts))
       changed!
       files
     end
 
     # Index a list of files
-    def index(files)
-      files.map do |file|
+    def index(fs)
+      fs.map do |file|
+        next if files.key?(file)
         [file, action.(file)]
-      end.to_h
+      end.compact.to_h
     end
 
     # @see #index
-    def index!(nf)
-      files.merge!(index(nf))
+    def index!(fs)
+      files.merge!(index(fs))
       changed!
       files
     end
 
+    # Prune nonexistant files from the index
+    def prune!
+      files.select! { |f,_| File.exist?(f) }.to_h
+    end
+
     # Watch directories for changes using the listen gem, when called make sure
     # to call #stop when done watching.
     def watch(dirs = @dirs, exts = @exts)
@@ -89,6 +97,11 @@ module FileIndexer
 
     private
 
+    # Makes sure a directory exists, and sanitizes it
+    def sanitize_dirpath(dir)
+      Pathname.new(File.expand_path(dir)).realpath.to_s
+    end
+
     # Glob all files with certain extensions in a directory
     def glob(dir, exts = @exts)
       Dir.glob("#{dir}**/*.{#{exts.join(',')}}")

data/pkg/file_indexer-1.0.0/lib/file_indexer.rb

@@ -0,0 +1,11 @@
+require_relative 'file_indexer/indexer'
+
+# Namespace for a simple file indexer
+module FileIndexer
+  VERSION = [1, 0, 1]
+
+  # Equivalent to FileIndexer::Indexer.new(dirs, exts, db_file, &action)
+  def self.new(dirs, exts, db_file, &action)
+    Indexer.new(dirs, exts, db_file, &action)
+  end
+end

data/pkg/file_indexer-1.0.0/lib/file_indexer/indexer.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+require 'listen'
+require 'observer'
+require 'pathname'
+
+module FileIndexer
+
+  # A simple file indexer, keeps a database of files and associated metadata
+  class Indexer
+    include Observable
+
+    # The directories that will be indexed
+    attr_accessor :dirs
+    # Only files with these extensions will be considered.  Just use '*' for all
+    # files.
+    attr_accessor :exts
+    # This is a Proc that will be called for every file and its result will be
+    # stored in the database
+    attr_accessor :action
+    # This is the actual database of files, it is a Hash
+    attr_accessor :files
+    # This is the filename which will be used as the persistent database
+    attr_reader :db_file
+
+    def initialize(dirs, exts, db_file, &action)
+      @dirs    = dirs.map { |d| sanitize_dirpath(d) }
+      @exts    = exts
+      @db_file = db_file
+      @action  = action
+
+      @files    = {}
+      @listener = nil
+
+      @files.merge!(read_db)
+      prune!
+      index_all!
+    end
+
+    # Index multiple directories
+    def index_all(dirs = @dirs, exts = @exts)
+      dirs.map { |dir| index(glob(dir, exts)) }.reduce(&:merge)
+    end
+
+    # @see #index_all
+    def index_all!(dirs = @dirs, exts = @exts)
+      files.merge!(index_all(dirs, exts))
+      changed!
+      files
+    end
+
+    # Index a list of files
+    def index(fs)
+      fs.map do |file|
+        next if files.key?(file)
+        [file, action.(file)]
+      end.compact.to_h
+    end
+
+    # @see #index
+    def index!(fs)
+      files.merge!(index(fs))
+      changed!
+      files
+    end
+
+    # Prune nonexistant files from the index
+    def prune!
+      files.select! { |f,_| File.exist?(f) }.to_h
+    end
+
+    # Watch directories for changes using the listen gem, when called make sure
+    # to call #stop when done watching.
+    def watch(dirs = @dirs, exts = @exts)
+      @listener =
+        Listen.to(*dirs, only: /\.(#{exts.join('|')})$/) do |mod, add, del|
+          del.each { |f| files.delete(f) }
+          index!(mod + add)
+        end
+      @listener.start
+    end
+
+    # Stop watching directories for changes
+    def stop
+      @listener&.stop
+    end
+
+    # Write the databse to disk
+    def write_db
+      File.open(db_file, 'w') { |f| f.puts(Marshal.dump(files)) }
+    end
+
+    # Read the database from disk
+    def read_db
+      return {} unless File.exist?(db_file)
+      File.open(db_file, 'r') { |f| Marshal.load(f.read) }
+    end
+
+    private
+
+    # Makes sure a directory exists, and sanitizes it
+    def sanitize_dirpath(dir)
+      Pathname.new(File.expand_path(dir)).realpath.to_s
+    end
+
+    # Glob all files with certain extensions in a directory
+    def glob(dir, exts = @exts)
+      Dir.glob("#{dir}**/*.{#{exts.join(',')}}")
+    end
+
+    # Notify observers of changes and rewrite the database file
+    def changed!(*args)
+      changed
+      write_db
+      notify_observers(*args)
+    end
+  end
+end

data/pkg/file_indexer-1.0.0/test/test_indexer.rb

@@ -0,0 +1,63 @@
+require 'test/unit'
+require_relative '../lib/file_indexer'
+
+DIRS     = [File.expand_path(File.join(__dir__, 'foo/')) + '/',
+            File.expand_path(File.join(__dir__, 'bar/')) + '/']
+FILES    = [File.join(DIRS[0], 'a.foo'),
+            File.join(DIRS[0], 'b.bar'),
+            File.join(DIRS[0], 'c.kek'),
+            File.join(DIRS[1], 'a.foo'),
+            File.join(DIRS[1], 'b.foo')]
+EXTS     = %w[foo bar]
+DB_FILE  = File.join(__dir__, 'db_file')
+TMP_FILE = File.join(DIRS[0], 'd.foo')
+
+class TestIndexer < Test::Unit::TestCase
+  def setup
+    DIRS.each { |d| FileUtils.mkdir_p(d) }
+    FILES.each { |f| FileUtils.touch(f) }
+    @indexer = FileIndexer.new(DIRS, EXTS, DB_FILE) do |f|
+      File.basename(f).upcase
+    end
+  end
+
+  def teardown
+    @indexer.stop
+    FileUtils.rm(DB_FILE) if File.exist?(DB_FILE)
+    FileUtils.rm(TMP_FILE) if File.exist?(TMP_FILE)
+    DIRS.each { |d| FileUtils.rm_rf(d) }
+  end
+
+  def test_glob
+    count = @indexer.dirs.map { |d| @indexer.send(:glob, d).length }.reduce(:+)
+    assert(4 == count, "Glob counts #{count} files, not 4")
+  end
+
+  def test_index_all
+    values = @indexer.index_all.values
+    assert(
+      ['A.FOO', 'B.BAR', 'B.FOO', 'A.FOO'] == values,
+      "Values are #{values}"
+    )
+  end
+
+  def test_index
+    assert(['A.FOO'] == @indexer.index([File.join(DIRS[0], 'a.foo')]).values)
+  end
+
+  def test_watch
+    @indexer.watch
+    sleep 0.1
+    FileUtils.touch(TMP_FILE)
+    sleep 1
+    assert(5 == @indexer.files.values.length)
+  end
+
+  def test_write_db
+    assert(true == File.exist?(DB_FILE))
+  end
+
+  def test_read_db
+    assert(4 == @indexer.read_db.values.length)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: file_indexer
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Stone Tickle
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-02 00:00:00.000000000 Z
+date: 2018-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: listen
@@ -39,6 +39,9 @@ files:
 - "./README.md"
 - "./lib/file_indexer.rb"
 - "./lib/file_indexer/indexer.rb"
+- "./pkg/file_indexer-1.0.0/lib/file_indexer.rb"
+- "./pkg/file_indexer-1.0.0/lib/file_indexer/indexer.rb"
+- "./pkg/file_indexer-1.0.0/test/test_indexer.rb"
 - "./test/test_indexer.rb"
 homepage: 
 licenses: