file_indexer 1.0.0 → 1.0.1
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7fc83ca7cc27629d1d6c8acbbc0e581158c4e960c5f0ab98c038eeff45118527
|
4
|
+
data.tar.gz: a1f07b862588d490953a6b9244fdc28e74caf821e8775abe028ccad20e991ff7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 753d11d8e8f3c96e0c496e21c7d5eab05b36e7cc55040e6480ccb16bde50119b5d34c99881cc8cc192c0e6c54ece347510758b87fe1f443b38909f9848aca564
|
7
|
+
data.tar.gz: ef2afddf154e51691352eff1943af6fa82b9876dd34a6359d12c66da677c853336a037c715cfea50e12c124cd0bfb471912c1268605afa8ba8c02246852765a8
|
data/lib/file_indexer.rb
CHANGED
data/lib/file_indexer/indexer.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
require 'listen'
|
3
3
|
require 'observer'
|
4
|
+
require 'pathname'
|
4
5
|
|
5
6
|
module FileIndexer
|
6
7
|
|
@@ -22,7 +23,7 @@ module FileIndexer
|
|
22
23
|
attr_reader :db_file
|
23
24
|
|
24
25
|
def initialize(dirs, exts, db_file, &action)
|
25
|
-
@dirs = dirs
|
26
|
+
@dirs = dirs.map { |d| sanitize_dirpath(d) }
|
26
27
|
@exts = exts
|
27
28
|
@db_file = db_file
|
28
29
|
@action = action
|
@@ -31,6 +32,7 @@ module FileIndexer
|
|
31
32
|
@listener = nil
|
32
33
|
|
33
34
|
@files.merge!(read_db)
|
35
|
+
prune!
|
34
36
|
index_all!
|
35
37
|
end
|
36
38
|
|
@@ -41,25 +43,31 @@ module FileIndexer
|
|
41
43
|
|
42
44
|
# @see #index_all
|
43
45
|
def index_all!(dirs = @dirs, exts = @exts)
|
44
|
-
|
46
|
+
files.merge!(index_all(dirs, exts))
|
45
47
|
changed!
|
46
48
|
files
|
47
49
|
end
|
48
50
|
|
49
51
|
# Index a list of files
|
50
|
-
def index(
|
51
|
-
|
52
|
+
def index(fs)
|
53
|
+
fs.map do |file|
|
54
|
+
next if files.key?(file)
|
52
55
|
[file, action.(file)]
|
53
|
-
end.to_h
|
56
|
+
end.compact.to_h
|
54
57
|
end
|
55
58
|
|
56
59
|
# @see #index
|
57
|
-
def index!(
|
58
|
-
files.merge!(index(
|
60
|
+
def index!(fs)
|
61
|
+
files.merge!(index(fs))
|
59
62
|
changed!
|
60
63
|
files
|
61
64
|
end
|
62
65
|
|
66
|
+
# Prune nonexistant files from the index
|
67
|
+
def prune!
|
68
|
+
files.select! { |f,_| File.exist?(f) }.to_h
|
69
|
+
end
|
70
|
+
|
63
71
|
# Watch directories for changes using the listen gem, when called make sure
|
64
72
|
# to call #stop when done watching.
|
65
73
|
def watch(dirs = @dirs, exts = @exts)
|
@@ -89,6 +97,11 @@ module FileIndexer
|
|
89
97
|
|
90
98
|
private
|
91
99
|
|
100
|
+
# Makes sure a directory exists, and sanitizes it
|
101
|
+
def sanitize_dirpath(dir)
|
102
|
+
Pathname.new(File.expand_path(dir)).realpath.to_s
|
103
|
+
end
|
104
|
+
|
92
105
|
# Glob all files with certain extensions in a directory
|
93
106
|
def glob(dir, exts = @exts)
|
94
107
|
Dir.glob("#{dir}**/*.{#{exts.join(',')}}")
|
@@ -0,0 +1,11 @@
|
|
1
|
+
require_relative 'file_indexer/indexer'
|
2
|
+
|
3
|
+
# Namespace for a simple file indexer
|
4
|
+
module FileIndexer
|
5
|
+
VERSION = [1, 0, 1]
|
6
|
+
|
7
|
+
# Equivalent to FileIndexer::Indexer.new(dirs, exts, db_file, &action)
|
8
|
+
def self.new(dirs, exts, db_file, &action)
|
9
|
+
Indexer.new(dirs, exts, db_file, &action)
|
10
|
+
end
|
11
|
+
end
|
@@ -0,0 +1,117 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'listen'
|
3
|
+
require 'observer'
|
4
|
+
require 'pathname'
|
5
|
+
|
6
|
+
module FileIndexer
|
7
|
+
|
8
|
+
# A simple file indexer, keeps a database of files and associated metadata
|
9
|
+
class Indexer
|
10
|
+
include Observable
|
11
|
+
|
12
|
+
# The directories that will be indexed
|
13
|
+
attr_accessor :dirs
|
14
|
+
# Only files with these extensions will be considered. Just use '*' for all
|
15
|
+
# files.
|
16
|
+
attr_accessor :exts
|
17
|
+
# This is a Proc that will be called for every file and its result will be
|
18
|
+
# stored in the database
|
19
|
+
attr_accessor :action
|
20
|
+
# This is the actual database of files, it is a Hash
|
21
|
+
attr_accessor :files
|
22
|
+
# This is the filename which will be used as the persistent database
|
23
|
+
attr_reader :db_file
|
24
|
+
|
25
|
+
def initialize(dirs, exts, db_file, &action)
|
26
|
+
@dirs = dirs.map { |d| sanitize_dirpath(d) }
|
27
|
+
@exts = exts
|
28
|
+
@db_file = db_file
|
29
|
+
@action = action
|
30
|
+
|
31
|
+
@files = {}
|
32
|
+
@listener = nil
|
33
|
+
|
34
|
+
@files.merge!(read_db)
|
35
|
+
prune!
|
36
|
+
index_all!
|
37
|
+
end
|
38
|
+
|
39
|
+
# Index multiple directories
|
40
|
+
def index_all(dirs = @dirs, exts = @exts)
|
41
|
+
dirs.map { |dir| index(glob(dir, exts)) }.reduce(&:merge)
|
42
|
+
end
|
43
|
+
|
44
|
+
# @see #index_all
|
45
|
+
def index_all!(dirs = @dirs, exts = @exts)
|
46
|
+
files.merge!(index_all(dirs, exts))
|
47
|
+
changed!
|
48
|
+
files
|
49
|
+
end
|
50
|
+
|
51
|
+
# Index a list of files
|
52
|
+
def index(fs)
|
53
|
+
fs.map do |file|
|
54
|
+
next if files.key?(file)
|
55
|
+
[file, action.(file)]
|
56
|
+
end.compact.to_h
|
57
|
+
end
|
58
|
+
|
59
|
+
# @see #index
|
60
|
+
def index!(fs)
|
61
|
+
files.merge!(index(fs))
|
62
|
+
changed!
|
63
|
+
files
|
64
|
+
end
|
65
|
+
|
66
|
+
# Prune nonexistant files from the index
|
67
|
+
def prune!
|
68
|
+
files.select! { |f,_| File.exist?(f) }.to_h
|
69
|
+
end
|
70
|
+
|
71
|
+
# Watch directories for changes using the listen gem, when called make sure
|
72
|
+
# to call #stop when done watching.
|
73
|
+
def watch(dirs = @dirs, exts = @exts)
|
74
|
+
@listener =
|
75
|
+
Listen.to(*dirs, only: /\.(#{exts.join('|')})$/) do |mod, add, del|
|
76
|
+
del.each { |f| files.delete(f) }
|
77
|
+
index!(mod + add)
|
78
|
+
end
|
79
|
+
@listener.start
|
80
|
+
end
|
81
|
+
|
82
|
+
# Stop watching directories for changes
|
83
|
+
def stop
|
84
|
+
@listener&.stop
|
85
|
+
end
|
86
|
+
|
87
|
+
# Write the databse to disk
|
88
|
+
def write_db
|
89
|
+
File.open(db_file, 'w') { |f| f.puts(Marshal.dump(files)) }
|
90
|
+
end
|
91
|
+
|
92
|
+
# Read the database from disk
|
93
|
+
def read_db
|
94
|
+
return {} unless File.exist?(db_file)
|
95
|
+
File.open(db_file, 'r') { |f| Marshal.load(f.read) }
|
96
|
+
end
|
97
|
+
|
98
|
+
private
|
99
|
+
|
100
|
+
# Makes sure a directory exists, and sanitizes it
|
101
|
+
def sanitize_dirpath(dir)
|
102
|
+
Pathname.new(File.expand_path(dir)).realpath.to_s
|
103
|
+
end
|
104
|
+
|
105
|
+
# Glob all files with certain extensions in a directory
|
106
|
+
def glob(dir, exts = @exts)
|
107
|
+
Dir.glob("#{dir}**/*.{#{exts.join(',')}}")
|
108
|
+
end
|
109
|
+
|
110
|
+
# Notify observers of changes and rewrite the database file
|
111
|
+
def changed!(*args)
|
112
|
+
changed
|
113
|
+
write_db
|
114
|
+
notify_observers(*args)
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
require 'test/unit'
|
2
|
+
require_relative '../lib/file_indexer'
|
3
|
+
|
4
|
+
DIRS = [File.expand_path(File.join(__dir__, 'foo/')) + '/',
|
5
|
+
File.expand_path(File.join(__dir__, 'bar/')) + '/']
|
6
|
+
FILES = [File.join(DIRS[0], 'a.foo'),
|
7
|
+
File.join(DIRS[0], 'b.bar'),
|
8
|
+
File.join(DIRS[0], 'c.kek'),
|
9
|
+
File.join(DIRS[1], 'a.foo'),
|
10
|
+
File.join(DIRS[1], 'b.foo')]
|
11
|
+
EXTS = %w[foo bar]
|
12
|
+
DB_FILE = File.join(__dir__, 'db_file')
|
13
|
+
TMP_FILE = File.join(DIRS[0], 'd.foo')
|
14
|
+
|
15
|
+
class TestIndexer < Test::Unit::TestCase
|
16
|
+
def setup
|
17
|
+
DIRS.each { |d| FileUtils.mkdir_p(d) }
|
18
|
+
FILES.each { |f| FileUtils.touch(f) }
|
19
|
+
@indexer = FileIndexer.new(DIRS, EXTS, DB_FILE) do |f|
|
20
|
+
File.basename(f).upcase
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def teardown
|
25
|
+
@indexer.stop
|
26
|
+
FileUtils.rm(DB_FILE) if File.exist?(DB_FILE)
|
27
|
+
FileUtils.rm(TMP_FILE) if File.exist?(TMP_FILE)
|
28
|
+
DIRS.each { |d| FileUtils.rm_rf(d) }
|
29
|
+
end
|
30
|
+
|
31
|
+
def test_glob
|
32
|
+
count = @indexer.dirs.map { |d| @indexer.send(:glob, d).length }.reduce(:+)
|
33
|
+
assert(4 == count, "Glob counts #{count} files, not 4")
|
34
|
+
end
|
35
|
+
|
36
|
+
def test_index_all
|
37
|
+
values = @indexer.index_all.values
|
38
|
+
assert(
|
39
|
+
['A.FOO', 'B.BAR', 'B.FOO', 'A.FOO'] == values,
|
40
|
+
"Values are #{values}"
|
41
|
+
)
|
42
|
+
end
|
43
|
+
|
44
|
+
def test_index
|
45
|
+
assert(['A.FOO'] == @indexer.index([File.join(DIRS[0], 'a.foo')]).values)
|
46
|
+
end
|
47
|
+
|
48
|
+
def test_watch
|
49
|
+
@indexer.watch
|
50
|
+
sleep 0.1
|
51
|
+
FileUtils.touch(TMP_FILE)
|
52
|
+
sleep 1
|
53
|
+
assert(5 == @indexer.files.values.length)
|
54
|
+
end
|
55
|
+
|
56
|
+
def test_write_db
|
57
|
+
assert(true == File.exist?(DB_FILE))
|
58
|
+
end
|
59
|
+
|
60
|
+
def test_read_db
|
61
|
+
assert(4 == @indexer.read_db.values.length)
|
62
|
+
end
|
63
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: file_indexer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Stone Tickle
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-05-
|
11
|
+
date: 2018-05-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: listen
|
@@ -39,6 +39,9 @@ files:
|
|
39
39
|
- "./README.md"
|
40
40
|
- "./lib/file_indexer.rb"
|
41
41
|
- "./lib/file_indexer/indexer.rb"
|
42
|
+
- "./pkg/file_indexer-1.0.0/lib/file_indexer.rb"
|
43
|
+
- "./pkg/file_indexer-1.0.0/lib/file_indexer/indexer.rb"
|
44
|
+
- "./pkg/file_indexer-1.0.0/test/test_indexer.rb"
|
42
45
|
- "./test/test_indexer.rb"
|
43
46
|
homepage:
|
44
47
|
licenses:
|