file_indexer 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: b0df350bb38bc48552894171e4ed305c67f152cf4bcac83f46af44911ff659f0
+  data.tar.gz: 59d9c6aa03fc9fe4e6bc4311c6a5295384bff30410cfe17e4b847c7d13618f02
+SHA512:
+  metadata.gz: a088e4f5918659b8f250ea09137a2b781b2b0978701b39783835c5afce08ea70586ab81f45af4e787e554c4b217e2ba26e5930d4a7db2b9e8aea62b0b186ac81
+  data.tar.gz: e806cec507fb7a9b8b8687e44671a03ae67c51e09c2398e9ce79de2b93ccc88096307a7d3f40556fe2eef3712c466c434f719c2e6c65501b4c1531010078031d

data/README.md

@@ -0,0 +1,34 @@
+# File Indexer
+
+A simple file indexer that allows you to keep track of all the files in a
+directory while also extracting information about those files.
+
+# What it can do:
+- Keep an index of files in a directory
+- Associate an object with each file, storing e.g. id3-tag data, md5 digests,
+  or anything else that is too costly to recompute over and over.
+- Keep the index synchronized to the file system by using the listen gem.
+- Use a persistent database file to avoid having to rescan the entire database
+  every time the program loads.
+
+# Usage:
+
+```ruby
+fi = FileIndexer.new(%w[~/music/], %w[mp3 wav], 'db') do |file|
+
+  # Use a mp3 tag reading library such as taglib-ruby
+
+  { artist: file.artist,
+    title:  file.track,
+    ...
+  }
+end
+
+fi.watch
+```
+# Documentation:
+
+To generate documentation with e.g. yard, simply
+
+    cd file_indexer
+    yard server

data/lib/file_indexer/indexer.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+require 'listen'
+require 'observer'
+
+module FileIndexer
+
+  # A simple file indexer, keeps a database of files and associated metadata
+  class Indexer
+    include Observable
+
+    # The directories that will be indexed
+    attr_accessor :dirs
+    # Only files with these extensions will be considered.  Just use '*' for all
+    # files.
+    attr_accessor :exts
+    # This is a Proc that will be called for every file and its result will be
+    # stored in the database
+    attr_accessor :action
+    # This is the actual database of files, it is a Hash
+    attr_accessor :files
+    # This is the filename which will be used as the persistent database
+    attr_reader :db_file
+
+    def initialize(dirs, exts, db_file, &action)
+      @dirs    = dirs
+      @exts    = exts
+      @db_file = db_file
+      @action  = action
+
+      @files    = {}
+      @listener = nil
+
+      @files.merge!(read_db)
+      index_all!
+    end
+
+    # Index multiple directories
+    def index_all(dirs = @dirs, exts = @exts)
+      dirs.map { |dir| index(glob(dir, exts)) }.reduce(&:merge)
+    end
+
+    # @see #index_all
+    def index_all!(dirs = @dirs, exts = @exts)
+      @files = index_all(dirs, exts)
+      changed!
+      files
+    end
+
+    # Index a list of files
+    def index(files)
+      files.map do |file|
+        [file, action.(file)]
+      end.to_h
+    end
+
+    # @see #index
+    def index!(nf)
+      files.merge!(index(nf))
+      changed!
+      files
+    end
+
+    # Watch directories for changes using the listen gem, when called make sure
+    # to call #stop when done watching.
+    def watch(dirs = @dirs, exts = @exts)
+      @listener =
+        Listen.to(*dirs, only: /\.(#{exts.join('|')})$/) do |mod, add, del|
+          del.each { |f| files.delete(f) }
+          index!(mod + add)
+        end
+      @listener.start
+    end
+
+    # Stop watching directories for changes
+    def stop
+      @listener&.stop
+    end
+
+    # Write the databse to disk
+    def write_db
+      File.open(db_file, 'w') { |f| f.puts(Marshal.dump(files)) }
+    end
+
+    # Read the database from disk
+    def read_db
+      return {} unless File.exist?(db_file)
+      File.open(db_file, 'r') { |f| Marshal.load(f.read) }
+    end
+
+    private
+
+    # Glob all files with certain extensions in a directory
+    def glob(dir, exts = @exts)
+      Dir.glob("#{dir}**/*.{#{exts.join(',')}}")
+    end
+
+    # Notify observers of changes and rewrite the database file
+    def changed!(*args)
+      changed
+      write_db
+      notify_observers(*args)
+    end
+  end
+end

data/lib/file_indexer.rb

@@ -0,0 +1,11 @@
+require_relative 'file_indexer/indexer'
+
+# Namespace for a simple file indexer
+module FileIndexer
+  VERSION = [1, 0, 0]
+
+  # Equivalent to FileIndexer::Indexer.new(dirs, exts, db_file, &action)
+  def self.new(dirs, exts, db_file, &action)
+    Indexer.new(dirs, exts, db_file, &action)
+  end
+end

data/test/test_indexer.rb

@@ -0,0 +1,63 @@
+require 'test/unit'
+require_relative '../lib/file_indexer'
+
+DIRS     = [File.expand_path(File.join(__dir__, 'foo/')) + '/',
+            File.expand_path(File.join(__dir__, 'bar/')) + '/']
+FILES    = [File.join(DIRS[0], 'a.foo'),
+            File.join(DIRS[0], 'b.bar'),
+            File.join(DIRS[0], 'c.kek'),
+            File.join(DIRS[1], 'a.foo'),
+            File.join(DIRS[1], 'b.foo')]
+EXTS     = %w[foo bar]
+DB_FILE  = File.join(__dir__, 'db_file')
+TMP_FILE = File.join(DIRS[0], 'd.foo')
+
+class TestIndexer < Test::Unit::TestCase
+  def setup
+    DIRS.each { |d| FileUtils.mkdir_p(d) }
+    FILES.each { |f| FileUtils.touch(f) }
+    @indexer = FileIndexer.new(DIRS, EXTS, DB_FILE) do |f|
+      File.basename(f).upcase
+    end
+  end
+
+  def teardown
+    @indexer.stop
+    FileUtils.rm(DB_FILE) if File.exist?(DB_FILE)
+    FileUtils.rm(TMP_FILE) if File.exist?(TMP_FILE)
+    DIRS.each { |d| FileUtils.rm_rf(d) }
+  end
+
+  def test_glob
+    count = @indexer.dirs.map { |d| @indexer.send(:glob, d).length }.reduce(:+)
+    assert(4 == count, "Glob counts #{count} files, not 4")
+  end
+
+  def test_index_all
+    values = @indexer.index_all.values
+    assert(
+      ['A.FOO', 'B.BAR', 'B.FOO', 'A.FOO'] == values,
+      "Values are #{values}"
+    )
+  end
+
+  def test_index
+    assert(['A.FOO'] == @indexer.index([File.join(DIRS[0], 'a.foo')]).values)
+  end
+
+  def test_watch
+    @indexer.watch
+    sleep 0.1
+    FileUtils.touch(TMP_FILE)
+    sleep 1
+    assert(5 == @indexer.files.values.length)
+  end
+
+  def test_write_db
+    assert(true == File.exist?(DB_FILE))
+  end
+
+  def test_read_db
+    assert(4 == @indexer.read_db.values.length)
+  end
+end

metadata

@@ -0,0 +1,67 @@
+--- !ruby/object:Gem::Specification
+name: file_indexer
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Stone Tickle
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-05-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: listen
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.1.5
+description: Keep track of files in a directory
+email: lattis@mochiro.moe
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- "./README.md"
+- "./lib/file_indexer.rb"
+- "./lib/file_indexer/indexer.rb"
+- "./test/test_indexer.rb"
+homepage: 
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: Simple file indexer
+test_files: []