taggata 0.0.2 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +15 -15
- data/bin/taggata +2 -45
- data/lib/taggata.rb +4 -4
- data/lib/taggata/cli.rb +102 -0
- data/lib/taggata/db.rb +75 -0
- data/lib/taggata/db_adapters.rb +6 -0
- data/lib/taggata/db_adapters/abstract.rb +45 -0
- data/lib/taggata/db_adapters/sequel.rb +75 -0
- data/lib/taggata/db_adapters/sequel_migrations/001_initial.rb +34 -0
- data/lib/taggata/parser/query.rb +26 -14
- data/lib/taggata/parser/tag.rb +15 -9
- data/lib/taggata/persistent.rb +10 -0
- data/lib/taggata/persistent/abstract.rb +53 -0
- data/lib/taggata/persistent/directory.rb +80 -0
- data/lib/taggata/persistent/file.rb +99 -0
- data/lib/taggata/persistent/file_tag.rb +48 -0
- data/lib/taggata/persistent/tag.rb +40 -0
- data/lib/taggata/persistent/with_parent.rb +11 -0
- data/lib/taggata/scanner.rb +70 -0
- data/lib/taggata/version.rb +1 -1
- data/taggata.gemspec +3 -0
- data/test/parser/query_parser_test.rb +6 -5
- data/test/parser/tag_parser_test.rb +3 -2
- data/test/{filesystem_scanner_test.rb → scanner_test.rb} +10 -5
- data/test/taggata_test_helper.rb +1 -6
- data/test/tagging_test.rb +58 -0
- metadata +81 -9
- data/lib/taggata/directory.rb +0 -49
- data/lib/taggata/file.rb +0 -33
- data/lib/taggata/filesystem_scanner.rb +0 -72
- data/lib/taggata/tag.rb +0 -20
@@ -0,0 +1,70 @@
|
|
1
|
+
module Taggata
|
2
|
+
class Scanner
|
3
|
+
|
4
|
+
attr_reader :db, :jobs, :progress
|
5
|
+
|
6
|
+
def initialize(db)
|
7
|
+
@db = db
|
8
|
+
@jobs = []
|
9
|
+
@progress = { :files => 0, :directories => 0 }
|
10
|
+
end
|
11
|
+
|
12
|
+
def report_header
|
13
|
+
puts "Files\tDirectories\tQueued"
|
14
|
+
end
|
15
|
+
|
16
|
+
def report
|
17
|
+
puts "#{progress[:files]}\t#{progress[:directories]}\t\t#{jobs.length}"
|
18
|
+
end
|
19
|
+
|
20
|
+
def process(dir)
|
21
|
+
report_header
|
22
|
+
db.transaction do
|
23
|
+
jobs << [dir.id, dir.name]
|
24
|
+
until jobs.empty?
|
25
|
+
do_job *jobs.shift
|
26
|
+
report
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def do_job(parent_id, path)
|
32
|
+
files, directories = Dir.glob(File.join(path, '*'))
|
33
|
+
.sort
|
34
|
+
.partition { |entry| ::File.file? entry }
|
35
|
+
save_missing files.map { |f| ::File.basename f },
|
36
|
+
parent_id,
|
37
|
+
Persistent::File unless files.empty?
|
38
|
+
save_missing directories.map { |f| ::File.basename f },
|
39
|
+
parent_id,
|
40
|
+
Persistent::Directory unless directories.empty?
|
41
|
+
progress[:files] += files.length
|
42
|
+
add_directory_jobs directories,
|
43
|
+
parent_id unless directories.empty?
|
44
|
+
progress[:directories] += 1
|
45
|
+
end
|
46
|
+
|
47
|
+
def save_missing(files, parent_id, klass)
|
48
|
+
in_db = find_in_db(klass, parent_id, files, :name)
|
49
|
+
to_save = (files - in_db).map do |basename|
|
50
|
+
{ :name => basename, :parent_id => parent_id }
|
51
|
+
end
|
52
|
+
db.adapter.db[klass.table].multi_insert(to_save)
|
53
|
+
end
|
54
|
+
|
55
|
+
def find_in_db(klass, parent_id, names, param)
|
56
|
+
db.adapter.db[klass.table]
|
57
|
+
.where(:parent_id => parent_id)
|
58
|
+
.where(:name => names)
|
59
|
+
.map(param)
|
60
|
+
end
|
61
|
+
|
62
|
+
def add_directory_jobs(dirs, parent_id)
|
63
|
+
ids = find_in_db Persistent::Directory,
|
64
|
+
parent_id,
|
65
|
+
dirs.map { |d| ::File.basename d },
|
66
|
+
:id
|
67
|
+
ids.zip(dirs).each { |job| jobs << [job.first, job.last] }
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
data/lib/taggata/version.rb
CHANGED
data/taggata.gemspec
CHANGED
@@ -21,10 +21,13 @@ Gem::Specification.new do |spec|
|
|
21
21
|
|
22
22
|
spec.add_dependency 'sequel', '~> 4.22.0', '>= 4.22.0'
|
23
23
|
spec.add_dependency 'sqlite3', '~> 1.3.10', '>= 1.3.10'
|
24
|
+
spec.add_dependency 'clamp', '~>1.0.0', '>= 1.0.0'
|
24
25
|
|
26
|
+
spec.add_development_dependency 'thor', '~> 0.0', '>= 0.0.0'
|
25
27
|
spec.add_development_dependency 'bundler', '~> 1.7'
|
26
28
|
spec.add_development_dependency 'rake', '~> 10.0'
|
27
29
|
spec.add_development_dependency 'minitest', '~> 5.6.1', '>= 5.6.1'
|
28
30
|
spec.add_development_dependency 'minitest-reporters', '~> 1.0.16', '>= 1.0.16'
|
31
|
+
spec.add_development_dependency 'pry', '~> 0.0', '>= 0.0.0'
|
29
32
|
spec.add_development_dependency 'mocha', '~> 1.1', '>= 1.1.0'
|
30
33
|
end
|
@@ -3,7 +3,8 @@ require 'taggata_test_helper'
|
|
3
3
|
module Taggata
|
4
4
|
module Parser
|
5
5
|
describe Query do
|
6
|
-
let(:
|
6
|
+
let(:db) { Db.new 'sqlite:/', DbAdapters::Sequel }
|
7
|
+
let(:parser) { ::Taggata::Parser::Query.new db }
|
7
8
|
|
8
9
|
it 'translates' do
|
9
10
|
parser.send(:translate, '&').must_equal :and
|
@@ -16,9 +17,9 @@ module Taggata
|
|
16
17
|
end
|
17
18
|
|
18
19
|
it 'applies' do
|
19
|
-
parser.
|
20
|
-
parser.
|
21
|
-
proc { parser.apply
|
20
|
+
parser.send(:apply, :and, [1, 2], [2, 3]).must_equal [2]
|
21
|
+
parser.send(:apply, :or, [1, 2], [2, 3]).must_equal [1, 2, 3]
|
22
|
+
proc { parser.send(:apply, 'q', [], []) }.must_raise RuntimeError
|
22
23
|
end
|
23
24
|
|
24
25
|
it 'converts to postfix' do
|
@@ -43,7 +44,7 @@ module Taggata
|
|
43
44
|
|
44
45
|
it 'tries to resolve the token' do
|
45
46
|
tag = mock.tap { |t| t.expects(:files).returns([1]) }
|
46
|
-
::Taggata::Tag.expects(:
|
47
|
+
::Taggata::Persistent::Tag.expects(:find_one).with(db, :name => '2014').returns(tag)
|
47
48
|
parser.send(:resolve, 'is:2014').must_equal([1])
|
48
49
|
end
|
49
50
|
end
|
@@ -3,10 +3,11 @@ require 'taggata_test_helper'
|
|
3
3
|
module Taggata
|
4
4
|
module Parser
|
5
5
|
describe Tag do
|
6
|
-
let(:
|
6
|
+
let(:db) { Db.new 'sqlite:/', DbAdapters::Sequel }
|
7
|
+
let(:parser) { ::Taggata::Parser::Tag.new db }
|
7
8
|
|
8
9
|
after do
|
9
|
-
::Taggata::Tag.
|
10
|
+
::Taggata::Persistent::Tag.destroy(db, {})
|
10
11
|
end
|
11
12
|
|
12
13
|
it 'parses' do
|
@@ -1,10 +1,11 @@
|
|
1
1
|
require 'taggata_test_helper'
|
2
2
|
|
3
3
|
module Taggata
|
4
|
-
describe
|
4
|
+
describe Scanner do
|
5
5
|
let(:workdir) { ::Dir.mktmpdir }
|
6
|
-
let(:
|
7
|
-
let(:
|
6
|
+
let(:db) { Db.new 'sqlite:/', DbAdapters::Sequel }
|
7
|
+
let(:scanner) { Scanner.new db }
|
8
|
+
let(:root) { Persistent::Directory.find_or_create(db, :name => workdir) }
|
8
9
|
let(:file) { mock }
|
9
10
|
|
10
11
|
before do
|
@@ -17,31 +18,35 @@ module Taggata
|
|
17
18
|
|
18
19
|
it 'process creates initial job' do
|
19
20
|
root
|
21
|
+
scanner.expects(:report_header)
|
20
22
|
scanner.expects(:do_job).with(root.id, root.name)
|
21
23
|
scanner.process(root)
|
22
24
|
end
|
23
25
|
|
24
26
|
it 'scans empty directory' do
|
27
|
+
scanner.expects(:report_header)
|
25
28
|
scanner.expects(:save_missing).never
|
26
29
|
scanner.expects(:add_directory_jobs).never
|
27
30
|
scanner.process(root)
|
28
31
|
end
|
29
32
|
|
30
33
|
it 'scans directory with files' do
|
34
|
+
scanner.expects(:report_header)
|
31
35
|
basenames = (1..5).map { |i| "file-#{i}" }
|
32
36
|
basenames.each do |name|
|
33
37
|
FileUtils.touch(::File.join(workdir, name))
|
34
38
|
end
|
35
|
-
scanner.expects(:save_missing).with(basenames, root.id, ::
|
39
|
+
scanner.expects(:save_missing).with(basenames, root.id, Persistent::File)
|
36
40
|
scanner.expects(:add_directory_jobs).never
|
37
41
|
scanner.process(root)
|
38
42
|
end
|
39
43
|
|
40
44
|
it 'scans subdirectories' do
|
45
|
+
scanner.expects(:report_header)
|
41
46
|
basenames = (1..5).map { |i| "file-#{i}" }
|
42
47
|
subdir_path = ::File.join(workdir, 'subdir')
|
43
48
|
FileUtils.mkdir_p(subdir_path)
|
44
|
-
Directory.find_or_create(:name => 'subdir',
|
49
|
+
Persistent::Directory.find_or_create(db, :name => 'subdir',
|
45
50
|
:parent_id => root.id)
|
46
51
|
basenames.each do |name|
|
47
52
|
FileUtils.touch(::File.join(subdir_path, name))
|
data/test/taggata_test_helper.rb
CHANGED
@@ -1,10 +1,5 @@
|
|
1
1
|
require 'sequel'
|
2
|
-
|
3
|
-
DB.create_table :file_tags do
|
4
|
-
foreign_key :tag_id, :tags
|
5
|
-
foreign_key :file_id, :files
|
6
|
-
end unless DB.table_exists? :file_tags
|
7
|
-
Sequel::Model.plugin(:schema)
|
2
|
+
|
8
3
|
require 'taggata'
|
9
4
|
require 'minitest/autorun'
|
10
5
|
require 'minitest/reporters'
|
@@ -0,0 +1,58 @@
|
|
1
|
+
require 'taggata_test_helper'
|
2
|
+
|
3
|
+
module Taggata
|
4
|
+
module Persistent
|
5
|
+
describe Directory do
|
6
|
+
let(:db) { Db.new 'sqlite:/', DbAdapters::Sequel }
|
7
|
+
|
8
|
+
it 'can be created as root' do
|
9
|
+
root = Directory.find_or_create db, :name => 'root'
|
10
|
+
root.id.must_equal 1
|
11
|
+
root.parent.must_be_nil
|
12
|
+
end
|
13
|
+
|
14
|
+
it 'can have children' do
|
15
|
+
root = Directory.find_or_create db, :name => 'root'
|
16
|
+
subdir = Directory.find_or_create db, :name => 'subdir', :parent_id => root.id
|
17
|
+
subdir.parent_id.must_equal root.id
|
18
|
+
file = File.find_or_create db, :name => 'file', :parent_id => subdir.id
|
19
|
+
file.path.must_equal "root/subdir/file"
|
20
|
+
subdir.files.length.must_equal 1
|
21
|
+
subdir.directories.length.must_equal 0
|
22
|
+
root.entries.length.must_equal 1
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
|
27
|
+
describe File do
|
28
|
+
let(:db) { Db.new 'sqlite:/', DbAdapters::Sequel }
|
29
|
+
let(:root) { Directory.find_or_create db, :name => 'root' }
|
30
|
+
|
31
|
+
it 'cannot be created without parent' do
|
32
|
+
Proc.new { File.find_or_create(db, :name => 'myfile') }.must_raise Sequel::NotNullConstraintViolation
|
33
|
+
File.find_or_create db, :name => 'myfile', :parent_id => root.id
|
34
|
+
end
|
35
|
+
|
36
|
+
it 'can be tagged' do
|
37
|
+
file = File.find_or_create db, :name => 'myfile', :parent_id => root.id
|
38
|
+
tag = Tag.find_or_create db, :name => 'mytag'
|
39
|
+
file.add_tags(tag)
|
40
|
+
file.tags.map(&:name).must_equal ['mytag']
|
41
|
+
file.add_tags_by_name('tag1', 'tag2', 'tag3')
|
42
|
+
file.tags.map(&:name).must_equal ['mytag', 'tag1', 'tag2', 'tag3']
|
43
|
+
end
|
44
|
+
|
45
|
+
it 'can have tags removed' do
|
46
|
+
file = File.find_or_create db, :name => 'myfile', :parent_id => root.id
|
47
|
+
tag = Tag.find_or_create db, :name => 'mytag'
|
48
|
+
file.add_tags tag
|
49
|
+
file.add_tags_by_name 'mytag2'
|
50
|
+
file.tags.length.must_equal 2
|
51
|
+
file.remove_tags tag
|
52
|
+
file.remove_tags_by_name 'mytag2'
|
53
|
+
file.tags.length.must_equal 0
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: taggata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Adam Ruzicka
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-07-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sequel
|
@@ -50,6 +50,46 @@ dependencies:
|
|
50
50
|
- - ">="
|
51
51
|
- !ruby/object:Gem::Version
|
52
52
|
version: 1.3.10
|
53
|
+
- !ruby/object:Gem::Dependency
|
54
|
+
name: clamp
|
55
|
+
requirement: !ruby/object:Gem::Requirement
|
56
|
+
requirements:
|
57
|
+
- - "~>"
|
58
|
+
- !ruby/object:Gem::Version
|
59
|
+
version: 1.0.0
|
60
|
+
- - ">="
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: 1.0.0
|
63
|
+
type: :runtime
|
64
|
+
prerelease: false
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - "~>"
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: 1.0.0
|
70
|
+
- - ">="
|
71
|
+
- !ruby/object:Gem::Version
|
72
|
+
version: 1.0.0
|
73
|
+
- !ruby/object:Gem::Dependency
|
74
|
+
name: thor
|
75
|
+
requirement: !ruby/object:Gem::Requirement
|
76
|
+
requirements:
|
77
|
+
- - "~>"
|
78
|
+
- !ruby/object:Gem::Version
|
79
|
+
version: '0.0'
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: 0.0.0
|
83
|
+
type: :development
|
84
|
+
prerelease: false
|
85
|
+
version_requirements: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0.0'
|
90
|
+
- - ">="
|
91
|
+
- !ruby/object:Gem::Version
|
92
|
+
version: 0.0.0
|
53
93
|
- !ruby/object:Gem::Dependency
|
54
94
|
name: bundler
|
55
95
|
requirement: !ruby/object:Gem::Requirement
|
@@ -118,6 +158,26 @@ dependencies:
|
|
118
158
|
- - ">="
|
119
159
|
- !ruby/object:Gem::Version
|
120
160
|
version: 1.0.16
|
161
|
+
- !ruby/object:Gem::Dependency
|
162
|
+
name: pry
|
163
|
+
requirement: !ruby/object:Gem::Requirement
|
164
|
+
requirements:
|
165
|
+
- - "~>"
|
166
|
+
- !ruby/object:Gem::Version
|
167
|
+
version: '0.0'
|
168
|
+
- - ">="
|
169
|
+
- !ruby/object:Gem::Version
|
170
|
+
version: 0.0.0
|
171
|
+
type: :development
|
172
|
+
prerelease: false
|
173
|
+
version_requirements: !ruby/object:Gem::Requirement
|
174
|
+
requirements:
|
175
|
+
- - "~>"
|
176
|
+
- !ruby/object:Gem::Version
|
177
|
+
version: '0.0'
|
178
|
+
- - ">="
|
179
|
+
- !ruby/object:Gem::Version
|
180
|
+
version: 0.0.0
|
121
181
|
- !ruby/object:Gem::Dependency
|
122
182
|
name: mocha
|
123
183
|
requirement: !ruby/object:Gem::Requirement
|
@@ -155,20 +215,31 @@ files:
|
|
155
215
|
- Rakefile
|
156
216
|
- bin/taggata
|
157
217
|
- lib/taggata.rb
|
218
|
+
- lib/taggata/cli.rb
|
158
219
|
- lib/taggata/constants.rb
|
159
|
-
- lib/taggata/
|
160
|
-
- lib/taggata/
|
161
|
-
- lib/taggata/
|
220
|
+
- lib/taggata/db.rb
|
221
|
+
- lib/taggata/db_adapters.rb
|
222
|
+
- lib/taggata/db_adapters/abstract.rb
|
223
|
+
- lib/taggata/db_adapters/sequel.rb
|
224
|
+
- lib/taggata/db_adapters/sequel_migrations/001_initial.rb
|
162
225
|
- lib/taggata/parser.rb
|
163
226
|
- lib/taggata/parser/query.rb
|
164
227
|
- lib/taggata/parser/tag.rb
|
165
|
-
- lib/taggata/
|
228
|
+
- lib/taggata/persistent.rb
|
229
|
+
- lib/taggata/persistent/abstract.rb
|
230
|
+
- lib/taggata/persistent/directory.rb
|
231
|
+
- lib/taggata/persistent/file.rb
|
232
|
+
- lib/taggata/persistent/file_tag.rb
|
233
|
+
- lib/taggata/persistent/tag.rb
|
234
|
+
- lib/taggata/persistent/with_parent.rb
|
235
|
+
- lib/taggata/scanner.rb
|
166
236
|
- lib/taggata/version.rb
|
167
237
|
- taggata.gemspec
|
168
|
-
- test/filesystem_scanner_test.rb
|
169
238
|
- test/parser/query_parser_test.rb
|
170
239
|
- test/parser/tag_parser_test.rb
|
240
|
+
- test/scanner_test.rb
|
171
241
|
- test/taggata_test_helper.rb
|
242
|
+
- test/tagging_test.rb
|
172
243
|
homepage: https://github.com/adamruzicka/taggata
|
173
244
|
licenses:
|
174
245
|
- BSD-2-Clause
|
@@ -189,12 +260,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
189
260
|
version: '0'
|
190
261
|
requirements: []
|
191
262
|
rubyforge_project:
|
192
|
-
rubygems_version: 2.4.
|
263
|
+
rubygems_version: 2.4.7
|
193
264
|
signing_key:
|
194
265
|
specification_version: 4
|
195
266
|
summary: Gem for scanning the filesystem and storing it in sqlite database with tagging
|
196
267
|
test_files:
|
197
|
-
- test/filesystem_scanner_test.rb
|
198
268
|
- test/parser/query_parser_test.rb
|
199
269
|
- test/parser/tag_parser_test.rb
|
270
|
+
- test/scanner_test.rb
|
200
271
|
- test/taggata_test_helper.rb
|
272
|
+
- test/tagging_test.rb
|
data/lib/taggata/directory.rb
DELETED
@@ -1,49 +0,0 @@
|
|
1
|
-
module Taggata
|
2
|
-
class Directory < Sequel::Model(:directories)
|
3
|
-
set_schema do
|
4
|
-
primary_key :id
|
5
|
-
String :name
|
6
|
-
foreign_key :parent_id, :directories
|
7
|
-
end
|
8
|
-
|
9
|
-
create_table unless table_exists?
|
10
|
-
|
11
|
-
one_to_many :directories,
|
12
|
-
:key => :parent_id,
|
13
|
-
:class => self
|
14
|
-
|
15
|
-
one_to_many :files,
|
16
|
-
:key => :parent_id,
|
17
|
-
:class => ::Taggata::File
|
18
|
-
|
19
|
-
many_to_one :parent,
|
20
|
-
:key => :parent_id,
|
21
|
-
:class => self
|
22
|
-
|
23
|
-
def entries
|
24
|
-
directories + files
|
25
|
-
end
|
26
|
-
|
27
|
-
# Scan children of this directory
|
28
|
-
def scan
|
29
|
-
scanner = ::Taggata::FilesystemScanner.new
|
30
|
-
scanner.process(self)
|
31
|
-
validate
|
32
|
-
end
|
33
|
-
|
34
|
-
def validate
|
35
|
-
missing = ::Taggata::Tag.find_or_create(:name => MISSING_TAG_NAME)
|
36
|
-
files.reject { |f| ::File.exist? f.path }.each { |f| f.add_tag missing }
|
37
|
-
directories.each(&:validate)
|
38
|
-
end
|
39
|
-
|
40
|
-
# Get full path of this directory
|
41
|
-
#
|
42
|
-
# @result full path of this directory
|
43
|
-
def path
|
44
|
-
parents = [self]
|
45
|
-
parents << parents.last.parent while parents.last.parent
|
46
|
-
::File.join(parents.reverse.map(&:name))
|
47
|
-
end
|
48
|
-
end
|
49
|
-
end
|