taggata 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +37 -0
- data/.rubocop.yml +8 -0
- data/.travis.yml +6 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +23 -0
- data/README.md +69 -0
- data/Rakefile +10 -0
- data/bin/taggata +42 -0
- data/lib/taggata.rb +11 -0
- data/lib/taggata/constants.rb +5 -0
- data/lib/taggata/directory.rb +49 -0
- data/lib/taggata/file.rb +33 -0
- data/lib/taggata/filesystem_scanner.rb +72 -0
- data/lib/taggata/parser.rb +4 -0
- data/lib/taggata/parser/query.rb +111 -0
- data/lib/taggata/parser/tag.rb +40 -0
- data/lib/taggata/tag.rb +20 -0
- data/lib/taggata/version.rb +3 -0
- data/taggata.gemspec +33 -0
- data/test/filesystem_scanner_test.rb +53 -0
- data/test/parser/query_parser_test.rb +51 -0
- data/test/parser/tag_parser_test.rb +33 -0
- data/test/taggata_test_helper.rb +13 -0
- metadata +240 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 2d7713cd81be332797275ebd869ea39818c825a8
|
4
|
+
data.tar.gz: 138c4eb01a7c07cad98c8522463474d912ab7023
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 37a36288180cb07f9ba8889b77c2b3d54ef5ce6cffcba14d970caa6f4322a7410cd577427db3d76f5f13f21d4cbad44334b8c0d7d772999bcfb8c9870c47aecb
|
7
|
+
data.tar.gz: 6ec36c4c738c0265910a200428c6cdec2ccf7468cf2363677446c9ee7d37caed6ddba901a59c2106511676af136c79c25113ec2dfae15f69b667e97172575659
|
data/.gitignore
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
*.gem
|
2
|
+
*.rbc
|
3
|
+
/.config
|
4
|
+
/coverage/
|
5
|
+
/InstalledFiles
|
6
|
+
/pkg/
|
7
|
+
/spec/reports/
|
8
|
+
/test/tmp/
|
9
|
+
/test/version_tmp/
|
10
|
+
/tmp/
|
11
|
+
|
12
|
+
## Specific to RubyMotion:
|
13
|
+
.dat*
|
14
|
+
.repl_history
|
15
|
+
build/
|
16
|
+
|
17
|
+
## Documentation cache and generated files:
|
18
|
+
/.yardoc/
|
19
|
+
/_yardoc/
|
20
|
+
/doc/
|
21
|
+
/rdoc/
|
22
|
+
|
23
|
+
## Environment normalisation:
|
24
|
+
/.bundle/
|
25
|
+
/vendor/bundle
|
26
|
+
/lib/bundler/man/
|
27
|
+
|
28
|
+
# for a library or gem, you might want to ignore these files since the code is
|
29
|
+
# intended to run in multiple environments; otherwise, check them in:
|
30
|
+
Gemfile.lock
|
31
|
+
# .ruby-version
|
32
|
+
# .ruby-gemset
|
33
|
+
|
34
|
+
# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
|
35
|
+
.rvmrc
|
36
|
+
|
37
|
+
*.sqlite
|
data/.rubocop.yml
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
Copyright (c) 2015, Adam Ruzicka
|
2
|
+
All rights reserved.
|
3
|
+
|
4
|
+
Redistribution and use in source and binary forms, with or without
|
5
|
+
modification, are permitted provided that the following conditions are met:
|
6
|
+
|
7
|
+
* Redistributions of source code must retain the above copyright notice, this
|
8
|
+
list of conditions and the following disclaimer.
|
9
|
+
|
10
|
+
* Redistributions in binary form must reproduce the above copyright notice,
|
11
|
+
this list of conditions and the following disclaimer in the documentation
|
12
|
+
and/or other materials provided with the distribution.
|
13
|
+
|
14
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
15
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
16
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
17
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
18
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
19
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
20
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
21
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
22
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
23
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
data/README.md
ADDED
@@ -0,0 +1,69 @@
|
|
1
|
+
# Taggata [](https://travis-ci.org/adamruzicka/regren)
|
2
|
+
|
3
|
+
Ruby gem for file tagging, it can:
|
4
|
+
- scan filesystem and store metadata in database
|
5
|
+
- tag stored entries
|
6
|
+
- lookup entries matching tag queries
|
7
|
+
- add and remove tags from entries
|
8
|
+
|
9
|
+
## Installation
|
10
|
+
Clone this repository:
|
11
|
+
|
12
|
+
$ git clone https://github.com/adamruzicka/taggata.git
|
13
|
+
|
14
|
+
And then execute:
|
15
|
+
|
16
|
+
$ bundle install
|
17
|
+
|
18
|
+
## Usage
|
19
|
+
Scan filesystem:
|
20
|
+
|
21
|
+
$ taggata $DATABASE scan $PATH_TO_SCAN
|
22
|
+
|
23
|
+
Search for entries:
|
24
|
+
|
25
|
+
$ taggata $DATABASE search "$SEARCH_QUERY"
|
26
|
+
|
27
|
+
Get count of matching entries:
|
28
|
+
|
29
|
+
$ taggata $DATABASE count "$SEARCH_QUERY"
|
30
|
+
|
31
|
+
Tag an entry:
|
32
|
+
|
33
|
+
$ taggata $DATABASE tag "$SEARCH_QUERY" "$TAG_QUERY"
|
34
|
+
|
35
|
+
Remove files matchine query:
|
36
|
+
|
37
|
+
$ taggata $DATABASE remove "$SEARCH_QUERY"
|
38
|
+
|
39
|
+
Query formats:
|
40
|
+
Search query has format of "$TYPE:$PARAMETER", where:
|
41
|
+
- $TYPE can be one of:
|
42
|
+
- ```is``` - searches for tag
|
43
|
+
- ```tag``` - the same as ```is```
|
44
|
+
- ```path``` - matches against absolute paths of files
|
45
|
+
- ```name``` - matches against name of the file
|
46
|
+
- $PARAMETER
|
47
|
+
- for ```is``` and ```tag``` it is a string
|
48
|
+
- for ```path``` and ```name``` it is a regular expression
|
49
|
+
- special cases
|
50
|
+
- ```missing``` - searches for files which are present in the database but not on filesystem
|
51
|
+
- ```untagged``` - searches for files without any tag
|
52
|
+
|
53
|
+
The search queries can be combined by using operators ```and``` and ```or``` and parentheses.
|
54
|
+
For example to get all files tagged as photos taken in year 2014 and 2015 one would issue:
|
55
|
+
|
56
|
+
$ taggata $DATABASE search "is:photo and ( is:2014 or is:2015 )"
|
57
|
+
|
58
|
+
Tag query has format of ```+$TAG_NAME``` or ```-$TAG_NAME```, one can specify more in one query
|
59
|
+
For example to tag all untagged files containing ```Photos``` in path with tags ```photo``` and ```to-backup``` one would issue:
|
60
|
+
|
61
|
+
$ taggata $DATABASE tag "untagged and path:Photos" "+photo +to-backup -default"
|
62
|
+
|
63
|
+
## Contributing
|
64
|
+
|
65
|
+
1. Fork it ( https://github.com/[my-github-username]/taggata/fork )
|
66
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
67
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
68
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
69
|
+
5. Create a new Pull Request
|
data/Rakefile
ADDED
data/bin/taggata
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'sequel'
|
4
|
+
DB = Sequel.connect("sqlite://#{ARGV[0]}")
|
5
|
+
|
6
|
+
DB.create_table :file_tags do
|
7
|
+
foreign_key :tag_id, :tags
|
8
|
+
foreign_key :file_id, :files
|
9
|
+
end unless DB.table_exists? :file_tags
|
10
|
+
|
11
|
+
Sequel::Model.plugin(:schema)
|
12
|
+
|
13
|
+
require 'taggata'
|
14
|
+
|
15
|
+
case ARGV[1]
|
16
|
+
when 'scan'
|
17
|
+
root = ::Taggata::Directory.find_or_create(:name => ARGV[2])
|
18
|
+
root.scan
|
19
|
+
when 'search'
|
20
|
+
result = ::Taggata::Parser::Query.parse(ARGV[2])
|
21
|
+
if result.empty?
|
22
|
+
puts "No files matching query '#{ARGV[2]}'"
|
23
|
+
else
|
24
|
+
result.each do |f|
|
25
|
+
puts f.path
|
26
|
+
end
|
27
|
+
end
|
28
|
+
when 'tag'
|
29
|
+
result = ::Taggata::Parser::Query.parse(ARGV[2])
|
30
|
+
tag_hash = ::Taggata::Parser::Tag.parse(ARGV[3])
|
31
|
+
result.each do |file|
|
32
|
+
tag_hash[:del].each { |tag| file.remove_tag tag }
|
33
|
+
tag_hash[:add].each { |tag| file.add_tag tag }
|
34
|
+
end
|
35
|
+
when 'remove'
|
36
|
+
files = ::Taggata::Parser::Query.parse(ARGV[2])
|
37
|
+
files.each(&:destroy)
|
38
|
+
when 'count'
|
39
|
+
puts ::Taggata::Parser::Query.parse(ARGV[2]).count
|
40
|
+
when 'pry'
|
41
|
+
require 'pry'; binding.pry
|
42
|
+
end
|
data/lib/taggata.rb
ADDED
@@ -0,0 +1,49 @@
|
|
1
|
+
module Taggata
|
2
|
+
class Directory < Sequel::Model(:directories)
|
3
|
+
set_schema do
|
4
|
+
primary_key :id
|
5
|
+
String :name
|
6
|
+
foreign_key :parent_id, :directories
|
7
|
+
end
|
8
|
+
|
9
|
+
create_table unless table_exists?
|
10
|
+
|
11
|
+
one_to_many :directories,
|
12
|
+
:key => :parent_id,
|
13
|
+
:class => self
|
14
|
+
|
15
|
+
one_to_many :files,
|
16
|
+
:key => :parent_id,
|
17
|
+
:class => ::Taggata::File
|
18
|
+
|
19
|
+
many_to_one :parent,
|
20
|
+
:key => :parent_id,
|
21
|
+
:class => self
|
22
|
+
|
23
|
+
def entries
|
24
|
+
directories + files
|
25
|
+
end
|
26
|
+
|
27
|
+
# Scan children of this directory
|
28
|
+
def scan
|
29
|
+
scanner = ::Taggata::FilesystemScanner.new
|
30
|
+
scanner.process(self)
|
31
|
+
validate
|
32
|
+
end
|
33
|
+
|
34
|
+
def validate
|
35
|
+
missing = ::Taggata::Tag.find_or_create(:name => MISSING_TAG_NAME)
|
36
|
+
files.reject { |f| ::File.exist? f.path }.each { |f| f.add_tag missing }
|
37
|
+
directories.each(&:validate)
|
38
|
+
end
|
39
|
+
|
40
|
+
# Get full path of this directory
|
41
|
+
#
|
42
|
+
# @result full path of this directory
|
43
|
+
def path
|
44
|
+
parents = [self]
|
45
|
+
parents << parents.last.parent while parents.last.parent
|
46
|
+
::File.join(parents.reverse.map(&:name))
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
data/lib/taggata/file.rb
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
module Taggata
|
2
|
+
class File < Sequel::Model(:files)
|
3
|
+
set_schema do
|
4
|
+
primary_key :id
|
5
|
+
String :name
|
6
|
+
foreign_key :parent_id, :directories, :on_delete => :set_null
|
7
|
+
end
|
8
|
+
|
9
|
+
def before_destroy
|
10
|
+
remove_all_tags
|
11
|
+
end
|
12
|
+
|
13
|
+
create_table unless table_exists?
|
14
|
+
|
15
|
+
require 'taggata/directory'
|
16
|
+
|
17
|
+
many_to_one :parent,
|
18
|
+
:key => :parent_id,
|
19
|
+
:class => ::Taggata::Directory
|
20
|
+
|
21
|
+
many_to_many :tags,
|
22
|
+
:left_id => :file_id,
|
23
|
+
:right_id => :tag_id,
|
24
|
+
:join_table => :file_tags
|
25
|
+
|
26
|
+
# Gets full path of the file
|
27
|
+
#
|
28
|
+
# @return String full path of the file
|
29
|
+
def path
|
30
|
+
::File.join(parent.path, name)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
module Taggata
|
2
|
+
class FilesystemScanner
|
3
|
+
# Initialize scanner's internal objects and default tag
|
4
|
+
def initialize
|
5
|
+
@jobs = []
|
6
|
+
@done_files = 0
|
7
|
+
@done_directories = 0
|
8
|
+
end
|
9
|
+
|
10
|
+
# Report progress
|
11
|
+
def report
|
12
|
+
print 'Done files/dirs - Queued: ',
|
13
|
+
"#{@done_files}/#{@done_directories} ",
|
14
|
+
"- #{@jobs.length}\n"
|
15
|
+
end
|
16
|
+
|
17
|
+
# Process directory at full path
|
18
|
+
#
|
19
|
+
# @param dir String name of the directory
|
20
|
+
# @param path String full path of the directory
|
21
|
+
def do_job(dir_id, path)
|
22
|
+
contents = Dir.glob("#{path}/*")
|
23
|
+
.reduce(::Taggata::File => [],
|
24
|
+
::Taggata::Directory => []) do |acc, cur|
|
25
|
+
key = ::File.file?(cur) ? ::Taggata::File : ::Taggata::Directory
|
26
|
+
acc.merge(key => acc[key].push(cur))
|
27
|
+
end
|
28
|
+
|
29
|
+
contents.each_pair do |klass, files|
|
30
|
+
save_missing files.map { |f| ::File.basename f },
|
31
|
+
dir_id,
|
32
|
+
klass unless files.empty?
|
33
|
+
end
|
34
|
+
@done_files += contents[::Taggata::File].length
|
35
|
+
add_directory_jobs contents[::Taggata::Directory],
|
36
|
+
dir_id unless contents[::Taggata::Directory].empty?
|
37
|
+
@done_directories += 1
|
38
|
+
end
|
39
|
+
|
40
|
+
def add_directory_jobs(dirs, parent_id)
|
41
|
+
ids = find_in_db ::Taggata::Directory,
|
42
|
+
parent_id,
|
43
|
+
dirs.map { |d| ::File.basename d },
|
44
|
+
:id
|
45
|
+
ids.zip(dirs).each { |job| @jobs << job }
|
46
|
+
end
|
47
|
+
|
48
|
+
def save_missing(files, parent_id, klass)
|
49
|
+
in_db = find_in_db(klass, parent_id, files, :name)
|
50
|
+
to_save = (files - in_db).map do |basename|
|
51
|
+
{ :name => basename, :parent_id => parent_id }
|
52
|
+
end
|
53
|
+
klass.dataset.multi_insert(to_save)
|
54
|
+
end
|
55
|
+
|
56
|
+
def find_in_db(klass, parent_id, names, param)
|
57
|
+
klass
|
58
|
+
.where(:parent_id => parent_id)
|
59
|
+
.where(:name => names)
|
60
|
+
.map(param)
|
61
|
+
end
|
62
|
+
|
63
|
+
# Breadth first search traversal through the filesystem tree
|
64
|
+
def process(dir)
|
65
|
+
@jobs << [dir.id, dir.name]
|
66
|
+
until @jobs.empty?
|
67
|
+
do_job(*@jobs.shift)
|
68
|
+
report
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
@@ -0,0 +1,111 @@
|
|
1
|
+
module Taggata
|
2
|
+
module Parser
|
3
|
+
class Query
|
4
|
+
def self.parse(query)
|
5
|
+
process(postfix(query))
|
6
|
+
end
|
7
|
+
|
8
|
+
private
|
9
|
+
|
10
|
+
# Resolves a terminal to a value
|
11
|
+
#
|
12
|
+
# @param token String the terminal token to resolve
|
13
|
+
# @result [::Taggata::File] list of files with this tag
|
14
|
+
def self.resolve(token)
|
15
|
+
type, name = token.split(':', 2)
|
16
|
+
case type.downcase
|
17
|
+
when 'is', 'tag'
|
18
|
+
::Taggata::Tag.files(:name => name)
|
19
|
+
when 'file', 'name'
|
20
|
+
File.all.select { |f| f.name[/#{name}/] }
|
21
|
+
when 'path'
|
22
|
+
File.all.select { |f| f.path[/#{name}/] }
|
23
|
+
when 'missing'
|
24
|
+
::Taggata::Tag.files(:name => MISSING_TAG_NAME)
|
25
|
+
when 'untagged'
|
26
|
+
ids = File.map(:id)
|
27
|
+
.select { |id| DB[:file_tags].where(:file_id => id).empty? }
|
28
|
+
File.where(:id => ids).all
|
29
|
+
else
|
30
|
+
fail "Unknown token type '#{type}'"
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
# Evaluates the input
|
35
|
+
#
|
36
|
+
# @param postfix [String] the input in postfix notation as array
|
37
|
+
# @return [::Taggata::File]
|
38
|
+
def self.process(postfix)
|
39
|
+
stack = []
|
40
|
+
postfix.each do |token|
|
41
|
+
if operator? token
|
42
|
+
op_b = stack.pop
|
43
|
+
op_a = stack.pop
|
44
|
+
stack << apply(token, op_a, op_b)
|
45
|
+
else
|
46
|
+
stack << resolve(token)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
stack.last
|
50
|
+
end
|
51
|
+
|
52
|
+
# Applies operator to operands op_A and op_B
|
53
|
+
#
|
54
|
+
# @param operator Symbol the operation(:and, :or) to apply to operands
|
55
|
+
# @param op_A [::Taggata::File] first operand
|
56
|
+
# @param op_B [::Taggata::File] second operand
|
57
|
+
# @result [::Taggata::File] result of applying operator to operands
|
58
|
+
def self.apply(operator, op_A, op_B)
|
59
|
+
case operator
|
60
|
+
when :and
|
61
|
+
op_A & op_B
|
62
|
+
when :or
|
63
|
+
op_A | op_B
|
64
|
+
else
|
65
|
+
fail "Unknown operator '#{operator}'"
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
# Converts string to postfix notation
|
70
|
+
#
|
71
|
+
# @param query String the query string
|
72
|
+
# @result [String] query in postfix notation as an array
|
73
|
+
def self.postfix(query)
|
74
|
+
postfix = []
|
75
|
+
operators = []
|
76
|
+
query.split.each do |token|
|
77
|
+
if operator? token
|
78
|
+
operators << translate(token)
|
79
|
+
elsif token == ')'
|
80
|
+
loop do
|
81
|
+
current = operators.pop
|
82
|
+
current == '(' ? break : postfix << current
|
83
|
+
end
|
84
|
+
else
|
85
|
+
postfix << token
|
86
|
+
end
|
87
|
+
end
|
88
|
+
operators.each { |op| postfix << op }
|
89
|
+
postfix
|
90
|
+
end
|
91
|
+
|
92
|
+
# Translates token to symbol or leaves it alone
|
93
|
+
#
|
94
|
+
# @param token String
|
95
|
+
# @result the token
|
96
|
+
def self.translate(token)
|
97
|
+
return :and if ['and', '&'].include? token.downcase
|
98
|
+
return :or if ['or', '|'].include? token.downcase
|
99
|
+
token
|
100
|
+
end
|
101
|
+
|
102
|
+
# Checks if token is an operator
|
103
|
+
#
|
104
|
+
# @param token String
|
105
|
+
# @result true/false
|
106
|
+
def self.operator?(token)
|
107
|
+
['&', '|', 'or', 'and', '(', :and, :or].include? token.downcase
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
module Taggata
|
2
|
+
module Parser
|
3
|
+
class Tag
|
4
|
+
# Parses give tagging string
|
5
|
+
#
|
6
|
+
# @param query String tagging string
|
7
|
+
# @return [Hash]
|
8
|
+
def self.parse(query)
|
9
|
+
result = { :add => [], :del => [] }
|
10
|
+
hash = query.split.reduce(result) do |acc, tag|
|
11
|
+
handle_tag(tag, acc)
|
12
|
+
end
|
13
|
+
dels = hash[:del].empty? ? [] : ::Taggata::Tag.where(:name => hash[:del]).all
|
14
|
+
adds = hash[:add].empty? ? [] : find_tags(hash[:add])
|
15
|
+
{ :add => adds, :del => dels }
|
16
|
+
end
|
17
|
+
|
18
|
+
private
|
19
|
+
|
20
|
+
def self.find_tags(names)
|
21
|
+
in_db = ::Taggata::Tag.where(:name => names).all
|
22
|
+
::Taggata::Tag
|
23
|
+
.dataset
|
24
|
+
.multi_insert((names - in_db).map { |name| { :name => name } })
|
25
|
+
::Taggata::Tag.where(:name => names).all
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.handle_tag(tag, result)
|
29
|
+
if tag.start_with?('-')
|
30
|
+
result[:del] << tag[1..-1]
|
31
|
+
elsif tag.start_with?('+')
|
32
|
+
result[:add] << tag[1..-1]
|
33
|
+
else
|
34
|
+
fail "Unknown tag specifier '#{tag}'"
|
35
|
+
end
|
36
|
+
result
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
data/lib/taggata/tag.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
module Taggata
|
2
|
+
class Tag < Sequel::Model
|
3
|
+
many_to_many :files,
|
4
|
+
:left_id => :tag_id,
|
5
|
+
:right_id => :file_id,
|
6
|
+
:join_table => :file_tags
|
7
|
+
|
8
|
+
set_schema do
|
9
|
+
primary_key :id
|
10
|
+
String :name
|
11
|
+
end
|
12
|
+
|
13
|
+
create_table unless table_exists?
|
14
|
+
|
15
|
+
def self.files(query)
|
16
|
+
tag = find(query)
|
17
|
+
tag.nil? ? [] : tag.files
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
data/taggata.gemspec
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'taggata/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = 'taggata'
|
8
|
+
spec.version = Taggata::VERSION
|
9
|
+
spec.authors = ['Adam Ruzicka']
|
10
|
+
spec.email = ['a.ruzicka@outlook.com']
|
11
|
+
spec.summary = 'Gem for scanning the filesystem and storing it in sqlite database with tagging'
|
12
|
+
# spec.summary = %q{TODO: Write a short summary. Required.}
|
13
|
+
# spec.description = %q{TODO: Write a longer description. Optional.}
|
14
|
+
spec.homepage = 'https://github.com/adamruzicka/taggata'
|
15
|
+
spec.license = 'BSD-2-Clause'
|
16
|
+
|
17
|
+
spec.files = `git ls-files -z`.split("\x0")
|
18
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
19
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
20
|
+
spec.require_paths = ['lib']
|
21
|
+
|
22
|
+
spec.add_dependency 'sequel', '~> 4.22.0', '>= 4.22.0'
|
23
|
+
spec.add_dependency 'sqlite3', '~> 1.3.10', '>= 1.3.10'
|
24
|
+
|
25
|
+
spec.add_development_dependency 'bundler', '~> 1.7'
|
26
|
+
spec.add_development_dependency 'rake', '~> 10.0'
|
27
|
+
spec.add_development_dependency 'minitest', '~> 5.6.1', '>= 5.6.1'
|
28
|
+
spec.add_development_dependency 'minitest-reporters', '~> 1.0.16', '>= 1.0.16'
|
29
|
+
spec.add_development_dependency 'mocha', '~> 1.1', '>= 1.1.0'
|
30
|
+
spec.add_development_dependency 'pry', '~> 0.1', '>= 0.0.1'
|
31
|
+
spec.add_development_dependency 'pry-coolline', '~> 0.1', '>= 0.0.1'
|
32
|
+
|
33
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
require 'taggata_test_helper'
|
2
|
+
|
3
|
+
module Taggata
|
4
|
+
describe FilesystemScanner do
|
5
|
+
let(:workdir) { ::Dir.mktmpdir }
|
6
|
+
let(:scanner) { FilesystemScanner.new }
|
7
|
+
let(:root) { Directory.find_or_create(:name => workdir) }
|
8
|
+
let(:file) { mock }
|
9
|
+
|
10
|
+
before do
|
11
|
+
scanner.expects(:report)
|
12
|
+
end
|
13
|
+
|
14
|
+
after do
|
15
|
+
FileUtils.rm_rf workdir
|
16
|
+
end
|
17
|
+
|
18
|
+
it 'process creates initial job' do
|
19
|
+
root
|
20
|
+
scanner.expects(:do_job).with(root.id, root.name)
|
21
|
+
scanner.process(root)
|
22
|
+
end
|
23
|
+
|
24
|
+
it 'scans empty directory' do
|
25
|
+
scanner.expects(:save_missing).never
|
26
|
+
scanner.expects(:add_directory_jobs).never
|
27
|
+
scanner.process(root)
|
28
|
+
end
|
29
|
+
|
30
|
+
it 'scans directory with files' do
|
31
|
+
basenames = (1..5).map { |i| "file-#{i}" }
|
32
|
+
basenames.each do |name|
|
33
|
+
FileUtils.touch(::File.join(workdir, name))
|
34
|
+
end
|
35
|
+
scanner.expects(:save_missing).with(basenames, root.id, ::Taggata::File)
|
36
|
+
scanner.expects(:add_directory_jobs).never
|
37
|
+
scanner.process(root)
|
38
|
+
end
|
39
|
+
|
40
|
+
it 'scans subdirectories' do
|
41
|
+
basenames = (1..5).map { |i| "file-#{i}" }
|
42
|
+
subdir_path = ::File.join(workdir, 'subdir')
|
43
|
+
FileUtils.mkdir_p(subdir_path)
|
44
|
+
Directory.find_or_create(:name => 'subdir',
|
45
|
+
:parent_id => root.id)
|
46
|
+
basenames.each do |name|
|
47
|
+
FileUtils.touch(::File.join(subdir_path, name))
|
48
|
+
end
|
49
|
+
scanner.expects(:add_directory_jobs).with([subdir_path], root.id)
|
50
|
+
scanner.process(root)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
require 'taggata_test_helper'
|
2
|
+
|
3
|
+
module Taggata
|
4
|
+
module Parser
|
5
|
+
describe Query do
|
6
|
+
let(:parser) { ::Taggata::Parser::Query }
|
7
|
+
|
8
|
+
it 'translates' do
|
9
|
+
parser.send(:translate, '&').must_equal :and
|
10
|
+
parser.send(:translate, 'and').must_equal :and
|
11
|
+
parser.send(:translate, 'AND').must_equal :and
|
12
|
+
parser.send(:translate, '|').must_equal :or
|
13
|
+
parser.send(:translate, 'or').must_equal :or
|
14
|
+
parser.send(:translate, 'OR').must_equal :or
|
15
|
+
parser.send(:translate, 'asdasd').must_equal 'asdasd'
|
16
|
+
end
|
17
|
+
|
18
|
+
it 'applies' do
|
19
|
+
parser.apply(:and, [1, 2], [2, 3]).must_equal [2]
|
20
|
+
parser.apply(:or, [1, 2], [2, 3]).must_equal [1, 2, 3]
|
21
|
+
proc { parser.apply('q', [], []) }.must_raise RuntimeError
|
22
|
+
end
|
23
|
+
|
24
|
+
it 'converts to postfix' do
|
25
|
+
parser.send(:postfix, '').must_equal []
|
26
|
+
parser.send(:postfix, 'is:2014').must_equal ['is:2014']
|
27
|
+
parser.send(:postfix, 'is:2014 and tag:2015')
|
28
|
+
.must_equal ['is:2014', 'tag:2015', :and]
|
29
|
+
parser.send(:postfix, 'is:2014 or tag:2015')
|
30
|
+
.must_equal ['is:2014', 'tag:2015', :or]
|
31
|
+
parser.send(:postfix, 'is:2014 and ( is:2015 or is:2016 )')
|
32
|
+
.must_equal ['is:2014', 'is:2015', 'is:2016', :or, :and]
|
33
|
+
end
|
34
|
+
|
35
|
+
it 'return type is always an array' do
|
36
|
+
parser.send(:resolve, 'is:test').must_be_instance_of Array
|
37
|
+
parser.send(:resolve, 'tag:test').must_be_instance_of Array
|
38
|
+
parser.send(:resolve, 'file:test').must_be_instance_of Array
|
39
|
+
parser.send(:resolve, 'path:test').must_be_instance_of Array
|
40
|
+
parser.send(:resolve, 'untagged').must_be_instance_of Array
|
41
|
+
parser.send(:resolve, 'missing').must_be_instance_of Array
|
42
|
+
end
|
43
|
+
|
44
|
+
it 'tries to resolve the token' do
|
45
|
+
tag = mock.tap { |t| t.expects(:files).returns([1]) }
|
46
|
+
::Taggata::Tag.expects(:find).with(:name => '2014').returns(tag)
|
47
|
+
parser.send(:resolve, 'is:2014').must_equal([1])
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'taggata_test_helper'
|
2
|
+
|
3
|
+
module Taggata
|
4
|
+
module Parser
|
5
|
+
describe Tag do
|
6
|
+
let(:parser) { ::Taggata::Parser::Tag }
|
7
|
+
|
8
|
+
after do
|
9
|
+
::Taggata::Tag.each(&:destroy)
|
10
|
+
end
|
11
|
+
|
12
|
+
it 'parses' do
|
13
|
+
parsed = parser.parse('+tag1')
|
14
|
+
parsed[:add].length.must_equal 1
|
15
|
+
parsed[:add].first.name.must_equal 'tag1'
|
16
|
+
parsed = parser.parse('+tag2 -tag1')
|
17
|
+
parsed[:add].map(&:name).must_equal ['tag2']
|
18
|
+
parsed[:del].map(&:name).must_equal ['tag1']
|
19
|
+
end
|
20
|
+
|
21
|
+
it 'fails when no + or - is provided' do
|
22
|
+
proc { parser.parse('badtag') }.must_raise RuntimeError
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'always returns a hash' do
|
26
|
+
parsed = parser.parse('+tag1 +tag2 -tag3 +tag4 -tag5')
|
27
|
+
parsed.must_be_instance_of Hash
|
28
|
+
parsed[:add].must_be_instance_of Array
|
29
|
+
parsed[:del].must_be_instance_of Array
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require 'sequel'
|
2
|
+
DB = Sequel.sqlite
|
3
|
+
DB.create_table :file_tags do
|
4
|
+
foreign_key :tag_id, :tags
|
5
|
+
foreign_key :file_id, :files
|
6
|
+
end unless DB.table_exists? :file_tags
|
7
|
+
Sequel::Model.plugin(:schema)
|
8
|
+
require 'taggata'
|
9
|
+
require 'minitest/autorun'
|
10
|
+
require 'minitest/reporters'
|
11
|
+
require 'fileutils'
|
12
|
+
require 'mocha/mini_test'
|
13
|
+
Minitest::Reporters.use! Minitest::Reporters::SpecReporter.new
|
metadata
ADDED
@@ -0,0 +1,240 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: taggata
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Adam Ruzicka
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-05-22 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: sequel
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 4.22.0
|
20
|
+
- - ">="
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: 4.22.0
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
requirements:
|
27
|
+
- - "~>"
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 4.22.0
|
30
|
+
- - ">="
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: 4.22.0
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: sqlite3
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - "~>"
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: 1.3.10
|
40
|
+
- - ">="
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
version: 1.3.10
|
43
|
+
type: :runtime
|
44
|
+
prerelease: false
|
45
|
+
version_requirements: !ruby/object:Gem::Requirement
|
46
|
+
requirements:
|
47
|
+
- - "~>"
|
48
|
+
- !ruby/object:Gem::Version
|
49
|
+
version: 1.3.10
|
50
|
+
- - ">="
|
51
|
+
- !ruby/object:Gem::Version
|
52
|
+
version: 1.3.10
|
53
|
+
- !ruby/object:Gem::Dependency
|
54
|
+
name: bundler
|
55
|
+
requirement: !ruby/object:Gem::Requirement
|
56
|
+
requirements:
|
57
|
+
- - "~>"
|
58
|
+
- !ruby/object:Gem::Version
|
59
|
+
version: '1.7'
|
60
|
+
type: :development
|
61
|
+
prerelease: false
|
62
|
+
version_requirements: !ruby/object:Gem::Requirement
|
63
|
+
requirements:
|
64
|
+
- - "~>"
|
65
|
+
- !ruby/object:Gem::Version
|
66
|
+
version: '1.7'
|
67
|
+
- !ruby/object:Gem::Dependency
|
68
|
+
name: rake
|
69
|
+
requirement: !ruby/object:Gem::Requirement
|
70
|
+
requirements:
|
71
|
+
- - "~>"
|
72
|
+
- !ruby/object:Gem::Version
|
73
|
+
version: '10.0'
|
74
|
+
type: :development
|
75
|
+
prerelease: false
|
76
|
+
version_requirements: !ruby/object:Gem::Requirement
|
77
|
+
requirements:
|
78
|
+
- - "~>"
|
79
|
+
- !ruby/object:Gem::Version
|
80
|
+
version: '10.0'
|
81
|
+
- !ruby/object:Gem::Dependency
|
82
|
+
name: minitest
|
83
|
+
requirement: !ruby/object:Gem::Requirement
|
84
|
+
requirements:
|
85
|
+
- - "~>"
|
86
|
+
- !ruby/object:Gem::Version
|
87
|
+
version: 5.6.1
|
88
|
+
- - ">="
|
89
|
+
- !ruby/object:Gem::Version
|
90
|
+
version: 5.6.1
|
91
|
+
type: :development
|
92
|
+
prerelease: false
|
93
|
+
version_requirements: !ruby/object:Gem::Requirement
|
94
|
+
requirements:
|
95
|
+
- - "~>"
|
96
|
+
- !ruby/object:Gem::Version
|
97
|
+
version: 5.6.1
|
98
|
+
- - ">="
|
99
|
+
- !ruby/object:Gem::Version
|
100
|
+
version: 5.6.1
|
101
|
+
- !ruby/object:Gem::Dependency
|
102
|
+
name: minitest-reporters
|
103
|
+
requirement: !ruby/object:Gem::Requirement
|
104
|
+
requirements:
|
105
|
+
- - "~>"
|
106
|
+
- !ruby/object:Gem::Version
|
107
|
+
version: 1.0.16
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: 1.0.16
|
111
|
+
type: :development
|
112
|
+
prerelease: false
|
113
|
+
version_requirements: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - "~>"
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: 1.0.16
|
118
|
+
- - ">="
|
119
|
+
- !ruby/object:Gem::Version
|
120
|
+
version: 1.0.16
|
121
|
+
- !ruby/object:Gem::Dependency
|
122
|
+
name: mocha
|
123
|
+
requirement: !ruby/object:Gem::Requirement
|
124
|
+
requirements:
|
125
|
+
- - "~>"
|
126
|
+
- !ruby/object:Gem::Version
|
127
|
+
version: '1.1'
|
128
|
+
- - ">="
|
129
|
+
- !ruby/object:Gem::Version
|
130
|
+
version: 1.1.0
|
131
|
+
type: :development
|
132
|
+
prerelease: false
|
133
|
+
version_requirements: !ruby/object:Gem::Requirement
|
134
|
+
requirements:
|
135
|
+
- - "~>"
|
136
|
+
- !ruby/object:Gem::Version
|
137
|
+
version: '1.1'
|
138
|
+
- - ">="
|
139
|
+
- !ruby/object:Gem::Version
|
140
|
+
version: 1.1.0
|
141
|
+
- !ruby/object:Gem::Dependency
|
142
|
+
name: pry
|
143
|
+
requirement: !ruby/object:Gem::Requirement
|
144
|
+
requirements:
|
145
|
+
- - "~>"
|
146
|
+
- !ruby/object:Gem::Version
|
147
|
+
version: '0.1'
|
148
|
+
- - ">="
|
149
|
+
- !ruby/object:Gem::Version
|
150
|
+
version: 0.0.1
|
151
|
+
type: :development
|
152
|
+
prerelease: false
|
153
|
+
version_requirements: !ruby/object:Gem::Requirement
|
154
|
+
requirements:
|
155
|
+
- - "~>"
|
156
|
+
- !ruby/object:Gem::Version
|
157
|
+
version: '0.1'
|
158
|
+
- - ">="
|
159
|
+
- !ruby/object:Gem::Version
|
160
|
+
version: 0.0.1
|
161
|
+
- !ruby/object:Gem::Dependency
|
162
|
+
name: pry-coolline
|
163
|
+
requirement: !ruby/object:Gem::Requirement
|
164
|
+
requirements:
|
165
|
+
- - "~>"
|
166
|
+
- !ruby/object:Gem::Version
|
167
|
+
version: '0.1'
|
168
|
+
- - ">="
|
169
|
+
- !ruby/object:Gem::Version
|
170
|
+
version: 0.0.1
|
171
|
+
type: :development
|
172
|
+
prerelease: false
|
173
|
+
version_requirements: !ruby/object:Gem::Requirement
|
174
|
+
requirements:
|
175
|
+
- - "~>"
|
176
|
+
- !ruby/object:Gem::Version
|
177
|
+
version: '0.1'
|
178
|
+
- - ">="
|
179
|
+
- !ruby/object:Gem::Version
|
180
|
+
version: 0.0.1
|
181
|
+
description:
|
182
|
+
email:
|
183
|
+
- a.ruzicka@outlook.com
|
184
|
+
executables:
|
185
|
+
- taggata
|
186
|
+
extensions: []
|
187
|
+
extra_rdoc_files: []
|
188
|
+
files:
|
189
|
+
- ".gitignore"
|
190
|
+
- ".rubocop.yml"
|
191
|
+
- ".travis.yml"
|
192
|
+
- Gemfile
|
193
|
+
- LICENSE.txt
|
194
|
+
- README.md
|
195
|
+
- Rakefile
|
196
|
+
- bin/taggata
|
197
|
+
- lib/taggata.rb
|
198
|
+
- lib/taggata/constants.rb
|
199
|
+
- lib/taggata/directory.rb
|
200
|
+
- lib/taggata/file.rb
|
201
|
+
- lib/taggata/filesystem_scanner.rb
|
202
|
+
- lib/taggata/parser.rb
|
203
|
+
- lib/taggata/parser/query.rb
|
204
|
+
- lib/taggata/parser/tag.rb
|
205
|
+
- lib/taggata/tag.rb
|
206
|
+
- lib/taggata/version.rb
|
207
|
+
- taggata.gemspec
|
208
|
+
- test/filesystem_scanner_test.rb
|
209
|
+
- test/parser/query_parser_test.rb
|
210
|
+
- test/parser/tag_parser_test.rb
|
211
|
+
- test/taggata_test_helper.rb
|
212
|
+
homepage: https://github.com/adamruzicka/taggata
|
213
|
+
licenses:
|
214
|
+
- BSD-2-Clause
|
215
|
+
metadata: {}
|
216
|
+
post_install_message:
|
217
|
+
rdoc_options: []
|
218
|
+
require_paths:
|
219
|
+
- lib
|
220
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
221
|
+
requirements:
|
222
|
+
- - ">="
|
223
|
+
- !ruby/object:Gem::Version
|
224
|
+
version: '0'
|
225
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
226
|
+
requirements:
|
227
|
+
- - ">="
|
228
|
+
- !ruby/object:Gem::Version
|
229
|
+
version: '0'
|
230
|
+
requirements: []
|
231
|
+
rubyforge_project:
|
232
|
+
rubygems_version: 2.4.5
|
233
|
+
signing_key:
|
234
|
+
specification_version: 4
|
235
|
+
summary: Gem for scanning the filesystem and storing it in sqlite database with tagging
|
236
|
+
test_files:
|
237
|
+
- test/filesystem_scanner_test.rb
|
238
|
+
- test/parser/query_parser_test.rb
|
239
|
+
- test/parser/tag_parser_test.rb
|
240
|
+
- test/taggata_test_helper.rb
|