librr 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +20 -0
- data/README.md +63 -0
- data/bin/daemon_run.rb~ +3 -0
- data/bin/librr +10 -0
- data/bin/librr~ +4 -0
- data/bin/test~ +6 -0
- data/lib/librr/cmd_client.rb +46 -0
- data/lib/librr/cmd_parser.rb +62 -0
- data/lib/librr/cmd_server.rb +82 -0
- data/lib/librr/configer.rb +33 -0
- data/lib/librr/delay_iterator.rb +18 -0
- data/lib/librr/dir_monitor.rb +116 -0
- data/lib/librr/indexer.rb +180 -0
- data/lib/librr/lib.rb +38 -0
- data/lib/librr/logger.rb +21 -0
- data/lib/librr/runner.rb +56 -0
- data/lib/librr/server_starter.rb +44 -0
- data/lib/librr/settings.rb +44 -0
- data/lib/librr/version.rb +3 -0
- data/lib/librr.rb +3 -0
- data/solr/makefile +2 -0
- data/solr/start.jar +0 -0
- metadata +196 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 7c1431a1d77217312122796724510c4e77fb34d0
|
4
|
+
data.tar.gz: 594faa85c5251131bc9f0679f1284daa321c7725
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 0a300c702916d50729106e6f08c72fb9fd1984f182eabf22fa72c68a783c32814073209cbd6cdc5bcf150224c766ae37ec1708bad6c2b29764d57c7a757b3456
|
7
|
+
data.tar.gz: a527e49131d00e7ab3a5542d55273db8fc0c9834eec178a6d6054052bec53060b772c76d70e1ed94132e688075c74bc15b4524c487be6f259bdb186703d61943
|
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2013 linjunhalida
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,63 @@
|
|
1
|
+
# librr
|
2
|
+
|
3
|
+
## about
|
4
|
+
|
5
|
+
It's a tool to index & search your local directory text files,
|
6
|
+
It use [solr](http://lucene.apache.org/solr/) for fulltext index.
|
7
|
+
|
8
|
+
## guide
|
9
|
+
|
10
|
+
First you need to add a file or directory:
|
11
|
+
|
12
|
+
```sh
|
13
|
+
librr add
|
14
|
+
```
|
15
|
+
|
16
|
+
## install
|
17
|
+
|
18
|
+
**System Requirements**: OSX or linux, Java 1.6 or greater, ruby gem system.
|
19
|
+
|
20
|
+
```
|
21
|
+
gem install librr
|
22
|
+
```
|
23
|
+
|
24
|
+
## usage
|
25
|
+
|
26
|
+
Start and stop background monitor process:
|
27
|
+
|
28
|
+
```sh
|
29
|
+
librr start
|
30
|
+
librr stop
|
31
|
+
```
|
32
|
+
|
33
|
+
It will start up automatically after first call to `librr search`,
|
34
|
+
You don't need to start it manually.
|
35
|
+
The background process name is: `librrd` and also start solr process.
|
36
|
+
|
37
|
+
|
38
|
+
Config search directories:
|
39
|
+
|
40
|
+
```sh
|
41
|
+
librr add ~/Dropbox/sync/docs
|
42
|
+
librr remove ~/Dropbox/sync/b
|
43
|
+
librr list
|
44
|
+
~/Dropbox/sync/docs
|
45
|
+
```
|
46
|
+
|
47
|
+
Using search:
|
48
|
+
|
49
|
+
```sh
|
50
|
+
librr search emacs
|
51
|
+
~/Dropbox/sync/docs/emacs.org:26: xxx emacs
|
52
|
+
~/Dropbox/sync/docs/gtd.org:230: bbb emacs
|
53
|
+
# or using sortcut:
|
54
|
+
librr s emacs
|
55
|
+
# set return result rows(default 30):
|
56
|
+
librr search emacs --rows 100
|
57
|
+
```
|
58
|
+
|
59
|
+
Schecdule reindex:
|
60
|
+
|
61
|
+
```sh
|
62
|
+
librr reindex [dir]
|
63
|
+
```
|
data/bin/daemon_run.rb~
ADDED
data/bin/librr
ADDED
data/bin/librr~
ADDED
data/bin/test~
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
require 'json'
|
3
|
+
|
4
|
+
require 'librr/logger'
|
5
|
+
require 'librr/server_starter'
|
6
|
+
|
7
|
+
|
8
|
+
class Librr::CmdClient
|
9
|
+
|
10
|
+
def initialize host, port
|
11
|
+
@host = host
|
12
|
+
@port = port
|
13
|
+
end
|
14
|
+
|
15
|
+
def check_start(sync=false)
|
16
|
+
begin
|
17
|
+
self.run_cmd(:ping)
|
18
|
+
return true
|
19
|
+
rescue Errno::ECONNREFUSED => e
|
20
|
+
end
|
21
|
+
|
22
|
+
ServerStarter.start_server(sync)
|
23
|
+
return false
|
24
|
+
end
|
25
|
+
|
26
|
+
def cmd cmd, params={}
|
27
|
+
begin
|
28
|
+
return self.run_cmd cmd, params
|
29
|
+
rescue Errno::ECONNREFUSED => e
|
30
|
+
end
|
31
|
+
|
32
|
+
puts "server not start, starting.."
|
33
|
+
ServerStarter.start_server(false)
|
34
|
+
ServerStarter.wait_for_server_started do
|
35
|
+
self.run_cmd cmd, **params
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def run_cmd cmd, params={}
|
40
|
+
params[:cmd] = cmd
|
41
|
+
url = '/cmd'
|
42
|
+
$logger.debug(:CmdClient){ "sending: #{params}" }
|
43
|
+
result = Net::HTTP.post_form(URI.parse("http://#{@host}:#{@port}#{url}"), params)
|
44
|
+
JSON.load(result.body)
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
require 'thor'
|
2
|
+
require 'librr/cmd_client'
|
3
|
+
require 'librr/settings'
|
4
|
+
|
5
|
+
|
6
|
+
class Librr::CmdParser < Thor
|
7
|
+
|
8
|
+
option :sync, type: :boolean
|
9
|
+
desc 'start [--sync]', 'start background process'
|
10
|
+
def start
|
11
|
+
if @@client.check_start(options[:sync])
|
12
|
+
puts 'server already started..'
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
desc 'stop', 'stop background process'
|
17
|
+
def stop
|
18
|
+
puts 'stopping..'
|
19
|
+
@@client.cmd(:stop) rescue nil
|
20
|
+
end
|
21
|
+
|
22
|
+
desc 'add DIR', 'add directory for indexing'
|
23
|
+
def add(dir)
|
24
|
+
puts "indexing: #{dir}"
|
25
|
+
@@client.cmd(:add, dir: File.expand_path(dir))
|
26
|
+
end
|
27
|
+
|
28
|
+
desc 'remove DIR', 'remove directory from indexing'
|
29
|
+
def remove(dir)
|
30
|
+
puts "removing: #{dir}"
|
31
|
+
@@client.cmd(:remove, dir: File.expand_path(dir))
|
32
|
+
end
|
33
|
+
|
34
|
+
desc 'list', 'list all indexed directories'
|
35
|
+
def list
|
36
|
+
puts @@client.cmd(:list)
|
37
|
+
end
|
38
|
+
|
39
|
+
desc "reindex", "reindex files"
|
40
|
+
def reindex
|
41
|
+
@@client.cmd(:reindex)
|
42
|
+
end
|
43
|
+
|
44
|
+
option :rows, type: :numeric, default: 20
|
45
|
+
option :all, type: :boolean
|
46
|
+
desc 'search STRING', 'search string'
|
47
|
+
def search(text)
|
48
|
+
puts "searching: #{text}"
|
49
|
+
results = @@client.cmd(:search, text: text, all: options[:all], rows: options[:rows])
|
50
|
+
if results.empty?
|
51
|
+
puts "find no result"
|
52
|
+
else
|
53
|
+
puts results.map{|v| v.join(":")}
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def self.run!
|
58
|
+
@@client = Librr::CmdClient.new('localhost', Settings.runner_port)
|
59
|
+
self.start(ARGV)
|
60
|
+
end
|
61
|
+
|
62
|
+
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
require 'eventmachine'
|
2
|
+
require 'evma_httpserver'
|
3
|
+
|
4
|
+
require 'json'
|
5
|
+
require 'rack'
|
6
|
+
|
7
|
+
|
8
|
+
class Librr::CmdServer
|
9
|
+
attr_accessor :monitor, :indexer
|
10
|
+
|
11
|
+
def init opts
|
12
|
+
self.monitor = opts[:monitor]
|
13
|
+
self.indexer = opts[:indexer]
|
14
|
+
CmdServerHandler.set_server(self)
|
15
|
+
end
|
16
|
+
|
17
|
+
def start(&block)
|
18
|
+
EventMachine.start_server "localhost", Settings.runner_port, CmdServerHandler
|
19
|
+
EM.add_timer(1){ block.call if block }
|
20
|
+
end
|
21
|
+
|
22
|
+
|
23
|
+
class CmdServerHandler < EM::Connection
|
24
|
+
include EM::HttpServer
|
25
|
+
include Librr::Logger::ClassLogger
|
26
|
+
|
27
|
+
def self.set_server(server)
|
28
|
+
@@server = server
|
29
|
+
end
|
30
|
+
|
31
|
+
def process_http_request
|
32
|
+
# puts @http_request_uri
|
33
|
+
response = EM::DelegatedHttpResponse.new(self)
|
34
|
+
response.status = 200
|
35
|
+
response.content_type 'application/json'
|
36
|
+
params = Rack::Utils.parse_nested_query(@http_post_content)
|
37
|
+
response.content = JSON.dump(self.handle_cmd(params))
|
38
|
+
response.send_response
|
39
|
+
end
|
40
|
+
|
41
|
+
def handle_cmd(params)
|
42
|
+
self.info "on receive: #{params.to_s}"
|
43
|
+
case params['cmd']
|
44
|
+
when 'ping'
|
45
|
+
'pong'
|
46
|
+
|
47
|
+
when 'stop'
|
48
|
+
puts "server stopping.."
|
49
|
+
EM.next_tick{
|
50
|
+
EM.stop
|
51
|
+
}
|
52
|
+
|
53
|
+
when 'add'
|
54
|
+
EM.next_tick{
|
55
|
+
@@server.monitor.add_directory(params['dir'])
|
56
|
+
}
|
57
|
+
|
58
|
+
when 'remove'
|
59
|
+
EM.next_tick{
|
60
|
+
@@server.monitor.remove_directory(params['dir'])
|
61
|
+
}
|
62
|
+
|
63
|
+
when 'list'
|
64
|
+
@@server.monitor.dirs.to_a
|
65
|
+
|
66
|
+
when 'reindex'
|
67
|
+
EM.next_tick{
|
68
|
+
@@server.monitor.reindex
|
69
|
+
}
|
70
|
+
|
71
|
+
when 'search'
|
72
|
+
@@server.indexer.search(params['text'], rows: params['rows'], all: params['all'])
|
73
|
+
|
74
|
+
else
|
75
|
+
raise Exception, "cmd unknown: #{params['cmd']}"
|
76
|
+
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
end
|
81
|
+
|
82
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'librr/settings'
|
2
|
+
|
3
|
+
module Configer
|
4
|
+
|
5
|
+
extend self
|
6
|
+
|
7
|
+
FILENAME = 'dir.conf'
|
8
|
+
|
9
|
+
def check_config_dir
|
10
|
+
conf_path = Settings::CONFIG_PATH
|
11
|
+
FileUtils.mkpath(conf_path) unless File.directory?(conf_path)
|
12
|
+
conf_path
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.load_dir_config
|
16
|
+
conf_path = self.check_config_dir
|
17
|
+
|
18
|
+
dc_file = Settings.in_dir(FILENAME)
|
19
|
+
if File.exists?(dc_file)
|
20
|
+
Set.new(File.read(dc_file).split("\n")).delete("")
|
21
|
+
else
|
22
|
+
[]
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.save_dir_config(config)
|
27
|
+
conf_path = self.check_config_dir
|
28
|
+
|
29
|
+
dc_file = Settings.in_dir(FILENAME)
|
30
|
+
File.write(dc_file, config.to_a.join("\n"))
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
class DelayIterator
|
2
|
+
def initialize(iter)
|
3
|
+
@iter = iter
|
4
|
+
end
|
5
|
+
|
6
|
+
def each(proc, finished=nil)
|
7
|
+
do_work = proc {
|
8
|
+
begin
|
9
|
+
item = @iter.next
|
10
|
+
proc.call(item)
|
11
|
+
EM.next_tick(&do_work)
|
12
|
+
rescue StopIteration
|
13
|
+
finished.call if finished
|
14
|
+
end
|
15
|
+
}
|
16
|
+
EM.next_tick(&do_work)
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,116 @@
|
|
1
|
+
require 'eventmachine'
|
2
|
+
require 'rb-fsevent'
|
3
|
+
require 'set'
|
4
|
+
|
5
|
+
require 'librr/logger'
|
6
|
+
require 'librr/configer'
|
7
|
+
|
8
|
+
|
9
|
+
class Librr::DirMonitor
|
10
|
+
include Librr::Logger::ClassLogger
|
11
|
+
|
12
|
+
attr_accessor :indexer, :dirs
|
13
|
+
|
14
|
+
def init opts
|
15
|
+
@pipe = nil
|
16
|
+
@new_start = false
|
17
|
+
@indexer = opts[:indexer]
|
18
|
+
|
19
|
+
self.dirs = Configer.load_dir_config
|
20
|
+
self.info "init dirs: #{self.dirs.to_a.to_s}"
|
21
|
+
end
|
22
|
+
|
23
|
+
def reindex
|
24
|
+
self.info "reindex"
|
25
|
+
@indexer.cleanup
|
26
|
+
self.dirs.each do |dir|
|
27
|
+
@indexer.index_directory(dir)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def add_directory(dir)
|
32
|
+
self.info "add dir: #{dir}"
|
33
|
+
@indexer.index_directory(dir)
|
34
|
+
self.dirs.add(dir)
|
35
|
+
Configer.save_dir_config(self.dirs)
|
36
|
+
self.info "save dir: #{self.dirs.to_a.to_s}"
|
37
|
+
self.start
|
38
|
+
end
|
39
|
+
|
40
|
+
def remove_directory(dir)
|
41
|
+
self.info "remove dir: #{dir}"
|
42
|
+
@indexer.remove_index_directory(dir)
|
43
|
+
self.dirs.delete(dir)
|
44
|
+
Configer.save_dir_config(self.dirs)
|
45
|
+
self.start
|
46
|
+
end
|
47
|
+
|
48
|
+
def post_init
|
49
|
+
@after_block.call if @after_block
|
50
|
+
end
|
51
|
+
|
52
|
+
def after_process_stop
|
53
|
+
self.start_process if @new_start
|
54
|
+
@new_start = false
|
55
|
+
end
|
56
|
+
|
57
|
+
def start &after_block
|
58
|
+
@after_block = after_block
|
59
|
+
|
60
|
+
if self.dirs.empty?
|
61
|
+
self.info "DIR empty, not start process."
|
62
|
+
self.post_init
|
63
|
+
return
|
64
|
+
end
|
65
|
+
|
66
|
+
if @pipe
|
67
|
+
@new_start = true
|
68
|
+
@pipe.close_connection
|
69
|
+
else
|
70
|
+
self.start_process
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def self.pid_file
|
75
|
+
Settings.in_dir('dir_watcher.pid')
|
76
|
+
end
|
77
|
+
|
78
|
+
def start_process
|
79
|
+
kill_process_by_file(self.class.pid_file)
|
80
|
+
|
81
|
+
cmd = [FSEvent.watcher_path] + ["--file-events"] + self.dirs.to_a
|
82
|
+
self.info "start process: #{cmd}"
|
83
|
+
@pipe = EM.popen(cmd, DirWatcher, self)
|
84
|
+
# TODO: write pid file
|
85
|
+
end
|
86
|
+
|
87
|
+
class DirWatcher < EventMachine::Connection
|
88
|
+
include Librr::Logger::ClassLogger
|
89
|
+
|
90
|
+
def initialize(monitor)
|
91
|
+
super
|
92
|
+
@monitor = monitor
|
93
|
+
end
|
94
|
+
|
95
|
+
def post_init
|
96
|
+
@monitor.post_init
|
97
|
+
end
|
98
|
+
|
99
|
+
def receive_data data
|
100
|
+
self.info "on receive data: #{data}"
|
101
|
+
changes = data.strip.split(':').map(&:strip).reject{|s| s == ''}
|
102
|
+
changes.each do |file|
|
103
|
+
@monitor.indexer.index_file(file)
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def unbind
|
108
|
+
self.info "dir monitor process stopped."
|
109
|
+
@monitor.after_process_stop
|
110
|
+
File.delete Librr::DirMonitor.pid_file rescue nil
|
111
|
+
end
|
112
|
+
|
113
|
+
end
|
114
|
+
|
115
|
+
end
|
116
|
+
|
@@ -0,0 +1,180 @@
|
|
1
|
+
require 'eventmachine'
|
2
|
+
# require 'rsolr-async' rescue nil
|
3
|
+
require 'rsolr'
|
4
|
+
|
5
|
+
|
6
|
+
require 'librr/lib'
|
7
|
+
require 'librr/settings'
|
8
|
+
require 'librr/delay_iterator'
|
9
|
+
|
10
|
+
|
11
|
+
class Librr::Indexer
|
12
|
+
include Librr::Logger::ClassLogger
|
13
|
+
|
14
|
+
attr_accessor :solr_started
|
15
|
+
|
16
|
+
SLICE_NUM = 300
|
17
|
+
|
18
|
+
def self.pid_file
|
19
|
+
Settings.in_dir('solr.pid')
|
20
|
+
end
|
21
|
+
|
22
|
+
def start &after_block
|
23
|
+
@after_block = after_block
|
24
|
+
|
25
|
+
kill_process_by_file(self.class.pid_file)
|
26
|
+
|
27
|
+
Dir.chdir File.join(Dir.pwd, 'solr') do
|
28
|
+
solr = 'java -jar start.jar'
|
29
|
+
solr_in, solr_out, solr_err = redirect_std do
|
30
|
+
EM.popen(solr, SolrManager)
|
31
|
+
# TODO: write pid file
|
32
|
+
end
|
33
|
+
EM.attach(solr_err, SolrOutHandler, self)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
module SolrManager
|
38
|
+
include Librr::Logger::ClassLogger
|
39
|
+
|
40
|
+
def post_init
|
41
|
+
self.info "start solr"
|
42
|
+
end
|
43
|
+
|
44
|
+
def receive_data data
|
45
|
+
self.info "receiving solr: #{data}"
|
46
|
+
end
|
47
|
+
|
48
|
+
def unbind
|
49
|
+
self.info "stop solr"
|
50
|
+
File.delete Librr::Indexer.pid_file rescue nil
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|
54
|
+
|
55
|
+
|
56
|
+
class SolrOutHandler < EventMachine::Connection
|
57
|
+
|
58
|
+
def initialize(indexer)
|
59
|
+
@indexer = indexer
|
60
|
+
end
|
61
|
+
|
62
|
+
def receive_data(data)
|
63
|
+
# File.open(Settings.in_dir('solr.log'), 'a+'){|f| f.write(data)}
|
64
|
+
if not @indexer.solr_started and data =~ /Started SocketConnector/
|
65
|
+
@indexer.after_start
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
70
|
+
|
71
|
+
|
72
|
+
def after_start
|
73
|
+
@solr_started = true
|
74
|
+
self.info 'after solr start'
|
75
|
+
|
76
|
+
@solr = RSolr.connect(
|
77
|
+
url: "http://localhost:#{Settings.solr_port}/solr",
|
78
|
+
read_timeout: 10, open_timeout: 10)
|
79
|
+
@after_block.call if @after_block
|
80
|
+
end
|
81
|
+
|
82
|
+
def run_solr &block
|
83
|
+
retry_times = 2
|
84
|
+
begin
|
85
|
+
block.call
|
86
|
+
rescue Net::ReadTimeout
|
87
|
+
retry_times -= 1
|
88
|
+
retry if retry_times >= 0
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
def cleanup
|
93
|
+
self.info 'cleanup'
|
94
|
+
self.run_solr {
|
95
|
+
@solr.delete_by_query '*:*'
|
96
|
+
@solr.commit
|
97
|
+
}
|
98
|
+
end
|
99
|
+
|
100
|
+
def index_directory(dir)
|
101
|
+
self.info "index dir: #{dir}"
|
102
|
+
files = Dir.glob(File.join(dir, "**/*"))
|
103
|
+
EM::Iterator.new(files)
|
104
|
+
.each(
|
105
|
+
proc { |file, iter|
|
106
|
+
if File.file?(file)
|
107
|
+
self.index_file(file){ iter.next }
|
108
|
+
else
|
109
|
+
iter.next
|
110
|
+
end
|
111
|
+
},
|
112
|
+
proc { self.info "index dir finished: #{dir}" }
|
113
|
+
)
|
114
|
+
end
|
115
|
+
|
116
|
+
def remove_index_directory(dir)
|
117
|
+
self.info "remove dir: #{dir}"
|
118
|
+
self.run_solr {
|
119
|
+
@solr.delete_by_query "filename:#{dir}*"
|
120
|
+
@solr.commit
|
121
|
+
}
|
122
|
+
end
|
123
|
+
|
124
|
+
def index_file(file, &block)
|
125
|
+
return if File.basename(file) =~ Settings.escape_files
|
126
|
+
|
127
|
+
self.run_solr {
|
128
|
+
@solr.delete_by_query "filename:#{file}"
|
129
|
+
@solr.commit
|
130
|
+
}
|
131
|
+
|
132
|
+
unless File.exists?(file)
|
133
|
+
self.info "remove index file: #{file}"
|
134
|
+
block.call if block
|
135
|
+
return
|
136
|
+
end
|
137
|
+
|
138
|
+
self.info "index file: #{file}"
|
139
|
+
f = File.open(file)
|
140
|
+
enum = f.each.each_slice(SLICE_NUM).each_with_index
|
141
|
+
self.info "file indexing...."
|
142
|
+
DelayIterator.new(enum)
|
143
|
+
.each(
|
144
|
+
proc { |lines, i|
|
145
|
+
data = lines.each_with_index.map do |line, j|
|
146
|
+
num = SLICE_NUM * i + j
|
147
|
+
line = fix_encoding(line).rstrip
|
148
|
+
{id: SecureRandom.uuid, filename: file, linenum: num, line: line}
|
149
|
+
end
|
150
|
+
|
151
|
+
self.run_solr {
|
152
|
+
@solr.add data
|
153
|
+
@solr.commit
|
154
|
+
}
|
155
|
+
|
156
|
+
self.info "working on lines: #{i*SLICE_NUM}"
|
157
|
+
},
|
158
|
+
proc {
|
159
|
+
f.close
|
160
|
+
block.call if block
|
161
|
+
}
|
162
|
+
)
|
163
|
+
end
|
164
|
+
|
165
|
+
def search(str, opts={})
|
166
|
+
self.info "search: #{str}"
|
167
|
+
|
168
|
+
rows = opts[:rows] || 30
|
169
|
+
rows = (2 ** 31 - 1) if opts[:all]
|
170
|
+
|
171
|
+
result = self.run_solr {
|
172
|
+
@solr.get 'select', params: {q: "line:#{str}", rows: rows}
|
173
|
+
}
|
174
|
+
|
175
|
+
result['response']['docs'].map do |row|
|
176
|
+
[row['filename'], row['linenum'], row['line']].flatten
|
177
|
+
end
|
178
|
+
end
|
179
|
+
|
180
|
+
end
|
data/lib/librr/lib.rb
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
def redirect_std
|
2
|
+
stdin = $stdin.dup
|
3
|
+
stdout = $stdout.dup
|
4
|
+
stderr = $stderr.dup
|
5
|
+
|
6
|
+
ri, wi = IO::pipe
|
7
|
+
ro, wo = IO::pipe
|
8
|
+
re, we = IO::pipe
|
9
|
+
|
10
|
+
$stdin.reopen ri
|
11
|
+
$stdout.reopen wo
|
12
|
+
$stderr.reopen we
|
13
|
+
|
14
|
+
yield
|
15
|
+
|
16
|
+
$stdin.reopen stdin
|
17
|
+
$stdout.reopen stdout
|
18
|
+
$stderr.reopen stderr
|
19
|
+
[wi, ro, re]
|
20
|
+
end
|
21
|
+
|
22
|
+
|
23
|
+
def fix_encoding text
|
24
|
+
# solution copy from:
|
25
|
+
# http://stackoverflow.com/questions/11375342/stringencode-not-fixing-invalid-byte-sequence-in-utf-8-error
|
26
|
+
text
|
27
|
+
.encode('UTF-16', undef: :replace, invalid: :replace, replace: "")
|
28
|
+
.encode('UTF-8')
|
29
|
+
end
|
30
|
+
|
31
|
+
def kill_process_by_file file
|
32
|
+
begin
|
33
|
+
pid = File.read(file).to_i
|
34
|
+
Process.kill 'TERM', pid if pid > 0
|
35
|
+
File.delete file
|
36
|
+
rescue
|
37
|
+
end
|
38
|
+
end
|
data/lib/librr/logger.rb
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'logger'
|
2
|
+
|
3
|
+
class Librr::Logger
|
4
|
+
|
5
|
+
module ClassLogger
|
6
|
+
def info(text)
|
7
|
+
$logger.info(self.class.name){ text }
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.create_logger
|
12
|
+
logger = Logger.new(STDOUT)
|
13
|
+
# logger.level = Logger::WARN
|
14
|
+
logger.level = Logger::DEBUG
|
15
|
+
logger
|
16
|
+
end
|
17
|
+
|
18
|
+
end
|
19
|
+
|
20
|
+
$logger ||= Librr::Logger.create_logger
|
21
|
+
|
data/lib/librr/runner.rb
ADDED
@@ -0,0 +1,56 @@
|
|
1
|
+
require 'eventmachine'
|
2
|
+
|
3
|
+
require 'librr/settings'
|
4
|
+
require 'librr/indexer'
|
5
|
+
require 'librr/dir_monitor'
|
6
|
+
require 'librr/cmd_server'
|
7
|
+
|
8
|
+
|
9
|
+
EventMachine.kqueue = true if EventMachine.kqueue?
|
10
|
+
|
11
|
+
|
12
|
+
class Librr::Runner
|
13
|
+
def run!
|
14
|
+
self.clear_pid
|
15
|
+
@stoping = false
|
16
|
+
|
17
|
+
EventMachine.run do
|
18
|
+
trap("SIGINT") do
|
19
|
+
return if @stoping
|
20
|
+
@stoping = true
|
21
|
+
|
22
|
+
EM.stop
|
23
|
+
puts "eventmachine graceful stops."
|
24
|
+
# todo commandline still show ^C?
|
25
|
+
self.clear_pid
|
26
|
+
end
|
27
|
+
|
28
|
+
indexer = Librr::Indexer.new
|
29
|
+
monitor = Librr::DirMonitor.new
|
30
|
+
server = Librr::CmdServer.new
|
31
|
+
|
32
|
+
monitor.init(indexer: indexer)
|
33
|
+
server.init(indexer: indexer, monitor: monitor)
|
34
|
+
|
35
|
+
indexer.start do
|
36
|
+
monitor.start do
|
37
|
+
server.start do
|
38
|
+
puts "server started"
|
39
|
+
self.write_pid
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def write_pid
|
48
|
+
filename = Settings::PID_FILE
|
49
|
+
File.open(filename, 'w+'){ |f| f.write(Process.pid.to_s) }
|
50
|
+
end
|
51
|
+
|
52
|
+
def clear_pid
|
53
|
+
filename = Settings::PID_FILE
|
54
|
+
File.delete(filename) rescue nil
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
# copy from gem daemons file: daemonize.rb
|
2
|
+
# todo: may has secruity risk
|
3
|
+
require 'librr/lib'
|
4
|
+
|
5
|
+
module ServerStarter
|
6
|
+
extend self
|
7
|
+
|
8
|
+
def run
|
9
|
+
require 'librr'
|
10
|
+
require 'librr/runner'
|
11
|
+
|
12
|
+
Librr::Runner.new.run!
|
13
|
+
end
|
14
|
+
|
15
|
+
def start_server(sync)
|
16
|
+
puts 'server starting..'
|
17
|
+
return self.run if sync
|
18
|
+
|
19
|
+
Process.fork do
|
20
|
+
sess_id = Process.setsid
|
21
|
+
Process.fork do
|
22
|
+
redirect_std do
|
23
|
+
$logger.info "daemon started."
|
24
|
+
self.run
|
25
|
+
end
|
26
|
+
exit
|
27
|
+
end
|
28
|
+
exit
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def wait_for_server_started &block
|
33
|
+
5.times.each do
|
34
|
+
sleep(2)
|
35
|
+
puts 'waiting for server starting..'
|
36
|
+
|
37
|
+
if File.exists?(Settings::PID_FILE)
|
38
|
+
return block.call if block
|
39
|
+
end
|
40
|
+
end
|
41
|
+
puts "server not starting, something is wrong."
|
42
|
+
exit
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
|
3
|
+
class Settings
|
4
|
+
|
5
|
+
CONFIG_PATH = File.expand_path('~/.librr/')
|
6
|
+
|
7
|
+
def self.in_dir(filename)
|
8
|
+
File.join(CONFIG_PATH, filename)
|
9
|
+
end
|
10
|
+
|
11
|
+
|
12
|
+
CONFIG_FILE = self.in_dir('config')
|
13
|
+
PID_FILE = self.in_dir('server.pid')
|
14
|
+
|
15
|
+
DEFAULTS = {
|
16
|
+
runner_port: 4512,
|
17
|
+
config_path: CONFIG_PATH,
|
18
|
+
escape_files: /[#~]$|^[\.#]/,
|
19
|
+
solr_port: 8901,
|
20
|
+
}
|
21
|
+
|
22
|
+
def self.reload
|
23
|
+
if File.exists?(CONFIG_FILE)
|
24
|
+
begin
|
25
|
+
@@config = YAML.load File.read(CONFIG_FILE)
|
26
|
+
raise unless @@config.kind_of?(Hash)
|
27
|
+
rescue
|
28
|
+
raise "config file format error: #{CONFIG_FILE}"
|
29
|
+
end
|
30
|
+
else
|
31
|
+
@@config = DEFAULTS.dup
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.method_missing(name)
|
36
|
+
self.reload unless defined?(@@config)
|
37
|
+
|
38
|
+
if @@config.include?(name)
|
39
|
+
@@config[name]
|
40
|
+
else
|
41
|
+
super
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
data/lib/librr.rb
ADDED
data/solr/makefile
ADDED
data/solr/start.jar
ADDED
Binary file
|
metadata
ADDED
@@ -0,0 +1,196 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: librr
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- linjunhalida
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2013-11-29 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: thor
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0.8'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ~>
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0.8'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: eventmachine
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: eventmachine_httpserver_update
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - '>='
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rsolr-async
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - '>='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: rb-fsevent
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - '>='
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - '>='
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: rack
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - '>='
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :runtime
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - '>='
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: rake
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - '>='
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - '>='
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: sass
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - '>='
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - '>='
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: rspec
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - '>='
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0'
|
132
|
+
type: :development
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - '>='
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '0'
|
139
|
+
description: |2
|
140
|
+
It is a tool to to index & search your text based documentation system.
|
141
|
+
It use solr for fulltext index.
|
142
|
+
email: linjunhalida@gmail.com
|
143
|
+
executables:
|
144
|
+
- librr
|
145
|
+
extensions: []
|
146
|
+
extra_rdoc_files:
|
147
|
+
- README.md
|
148
|
+
- LICENSE
|
149
|
+
files:
|
150
|
+
- README.md
|
151
|
+
- LICENSE
|
152
|
+
- bin/daemon_run.rb~
|
153
|
+
- bin/librr
|
154
|
+
- bin/librr~
|
155
|
+
- bin/test~
|
156
|
+
- lib/librr/cmd_client.rb
|
157
|
+
- lib/librr/cmd_parser.rb
|
158
|
+
- lib/librr/cmd_server.rb
|
159
|
+
- lib/librr/configer.rb
|
160
|
+
- lib/librr/delay_iterator.rb
|
161
|
+
- lib/librr/dir_monitor.rb
|
162
|
+
- lib/librr/indexer.rb
|
163
|
+
- lib/librr/lib.rb
|
164
|
+
- lib/librr/logger.rb
|
165
|
+
- lib/librr/runner.rb
|
166
|
+
- lib/librr/server_starter.rb
|
167
|
+
- lib/librr/settings.rb
|
168
|
+
- lib/librr/version.rb
|
169
|
+
- lib/librr.rb
|
170
|
+
- solr/makefile
|
171
|
+
- solr/start.jar
|
172
|
+
homepage: https://github.com/halida/librr
|
173
|
+
licenses:
|
174
|
+
- MIT
|
175
|
+
metadata: {}
|
176
|
+
post_install_message:
|
177
|
+
rdoc_options: []
|
178
|
+
require_paths:
|
179
|
+
- lib
|
180
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
181
|
+
requirements:
|
182
|
+
- - '>='
|
183
|
+
- !ruby/object:Gem::Version
|
184
|
+
version: 1.9.3
|
185
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
186
|
+
requirements:
|
187
|
+
- - '>='
|
188
|
+
- !ruby/object:Gem::Version
|
189
|
+
version: '0'
|
190
|
+
requirements: []
|
191
|
+
rubyforge_project:
|
192
|
+
rubygems_version: 2.1.11
|
193
|
+
signing_key:
|
194
|
+
specification_version: 4
|
195
|
+
summary: line based personal documentation search system.
|
196
|
+
test_files: []
|