librr 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/LICENSE +20 -0
- data/README.md +63 -0
- data/bin/daemon_run.rb~ +3 -0
- data/bin/librr +10 -0
- data/bin/librr~ +4 -0
- data/bin/test~ +6 -0
- data/lib/librr/cmd_client.rb +46 -0
- data/lib/librr/cmd_parser.rb +62 -0
- data/lib/librr/cmd_server.rb +82 -0
- data/lib/librr/configer.rb +33 -0
- data/lib/librr/delay_iterator.rb +18 -0
- data/lib/librr/dir_monitor.rb +116 -0
- data/lib/librr/indexer.rb +180 -0
- data/lib/librr/lib.rb +38 -0
- data/lib/librr/logger.rb +21 -0
- data/lib/librr/runner.rb +56 -0
- data/lib/librr/server_starter.rb +44 -0
- data/lib/librr/settings.rb +44 -0
- data/lib/librr/version.rb +3 -0
- data/lib/librr.rb +3 -0
- data/solr/makefile +2 -0
- data/solr/start.jar +0 -0
- metadata +196 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 7c1431a1d77217312122796724510c4e77fb34d0
|
4
|
+
data.tar.gz: 594faa85c5251131bc9f0679f1284daa321c7725
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 0a300c702916d50729106e6f08c72fb9fd1984f182eabf22fa72c68a783c32814073209cbd6cdc5bcf150224c766ae37ec1708bad6c2b29764d57c7a757b3456
|
7
|
+
data.tar.gz: a527e49131d00e7ab3a5542d55273db8fc0c9834eec178a6d6054052bec53060b772c76d70e1ed94132e688075c74bc15b4524c487be6f259bdb186703d61943
|
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2013 linjunhalida
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,63 @@
|
|
1
|
+
# librr
|
2
|
+
|
3
|
+
## about
|
4
|
+
|
5
|
+
It's a tool to index & search your local directory text files,
|
6
|
+
It use [solr](http://lucene.apache.org/solr/) for fulltext index.
|
7
|
+
|
8
|
+
## guide
|
9
|
+
|
10
|
+
First you need to add a file or directory:
|
11
|
+
|
12
|
+
```sh
|
13
|
+
librr add
|
14
|
+
```
|
15
|
+
|
16
|
+
## install
|
17
|
+
|
18
|
+
**System Requirements**: OSX or linux, Java 1.6 or greater, ruby gem system.
|
19
|
+
|
20
|
+
```
|
21
|
+
gem install librr
|
22
|
+
```
|
23
|
+
|
24
|
+
## usage
|
25
|
+
|
26
|
+
Start and stop background monitor process:
|
27
|
+
|
28
|
+
```sh
|
29
|
+
librr start
|
30
|
+
librr stop
|
31
|
+
```
|
32
|
+
|
33
|
+
It will start up automatically after first call to `librr search`,
|
34
|
+
You don't need to start it manually.
|
35
|
+
The background process name is: `librrd` and also start solr process.
|
36
|
+
|
37
|
+
|
38
|
+
Config search directories:
|
39
|
+
|
40
|
+
```sh
|
41
|
+
librr add ~/Dropbox/sync/docs
|
42
|
+
librr remove ~/Dropbox/sync/b
|
43
|
+
librr list
|
44
|
+
~/Dropbox/sync/docs
|
45
|
+
```
|
46
|
+
|
47
|
+
Using search:
|
48
|
+
|
49
|
+
```sh
|
50
|
+
librr search emacs
|
51
|
+
~/Dropbox/sync/docs/emacs.org:26: xxx emacs
|
52
|
+
~/Dropbox/sync/docs/gtd.org:230: bbb emacs
|
53
|
+
# or using sortcut:
|
54
|
+
librr s emacs
|
55
|
+
# set return result rows(default 30):
|
56
|
+
librr search emacs --rows 100
|
57
|
+
```
|
58
|
+
|
59
|
+
Schecdule reindex:
|
60
|
+
|
61
|
+
```sh
|
62
|
+
librr reindex [dir]
|
63
|
+
```
|
data/bin/daemon_run.rb~
ADDED
data/bin/librr
ADDED
data/bin/librr~
ADDED
data/bin/test~
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
require 'json'
|
3
|
+
|
4
|
+
require 'librr/logger'
|
5
|
+
require 'librr/server_starter'
|
6
|
+
|
7
|
+
|
8
|
+
class Librr::CmdClient
|
9
|
+
|
10
|
+
def initialize host, port
|
11
|
+
@host = host
|
12
|
+
@port = port
|
13
|
+
end
|
14
|
+
|
15
|
+
def check_start(sync=false)
|
16
|
+
begin
|
17
|
+
self.run_cmd(:ping)
|
18
|
+
return true
|
19
|
+
rescue Errno::ECONNREFUSED => e
|
20
|
+
end
|
21
|
+
|
22
|
+
ServerStarter.start_server(sync)
|
23
|
+
return false
|
24
|
+
end
|
25
|
+
|
26
|
+
def cmd cmd, params={}
|
27
|
+
begin
|
28
|
+
return self.run_cmd cmd, params
|
29
|
+
rescue Errno::ECONNREFUSED => e
|
30
|
+
end
|
31
|
+
|
32
|
+
puts "server not start, starting.."
|
33
|
+
ServerStarter.start_server(false)
|
34
|
+
ServerStarter.wait_for_server_started do
|
35
|
+
self.run_cmd cmd, **params
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def run_cmd cmd, params={}
|
40
|
+
params[:cmd] = cmd
|
41
|
+
url = '/cmd'
|
42
|
+
$logger.debug(:CmdClient){ "sending: #{params}" }
|
43
|
+
result = Net::HTTP.post_form(URI.parse("http://#{@host}:#{@port}#{url}"), params)
|
44
|
+
JSON.load(result.body)
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
require 'thor'
|
2
|
+
require 'librr/cmd_client'
|
3
|
+
require 'librr/settings'
|
4
|
+
|
5
|
+
|
6
|
+
class Librr::CmdParser < Thor
|
7
|
+
|
8
|
+
option :sync, type: :boolean
|
9
|
+
desc 'start [--sync]', 'start background process'
|
10
|
+
def start
|
11
|
+
if @@client.check_start(options[:sync])
|
12
|
+
puts 'server already started..'
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
desc 'stop', 'stop background process'
|
17
|
+
def stop
|
18
|
+
puts 'stopping..'
|
19
|
+
@@client.cmd(:stop) rescue nil
|
20
|
+
end
|
21
|
+
|
22
|
+
desc 'add DIR', 'add directory for indexing'
|
23
|
+
def add(dir)
|
24
|
+
puts "indexing: #{dir}"
|
25
|
+
@@client.cmd(:add, dir: File.expand_path(dir))
|
26
|
+
end
|
27
|
+
|
28
|
+
desc 'remove DIR', 'remove directory from indexing'
|
29
|
+
def remove(dir)
|
30
|
+
puts "removing: #{dir}"
|
31
|
+
@@client.cmd(:remove, dir: File.expand_path(dir))
|
32
|
+
end
|
33
|
+
|
34
|
+
desc 'list', 'list all indexed directories'
|
35
|
+
def list
|
36
|
+
puts @@client.cmd(:list)
|
37
|
+
end
|
38
|
+
|
39
|
+
desc "reindex", "reindex files"
|
40
|
+
def reindex
|
41
|
+
@@client.cmd(:reindex)
|
42
|
+
end
|
43
|
+
|
44
|
+
option :rows, type: :numeric, default: 20
|
45
|
+
option :all, type: :boolean
|
46
|
+
desc 'search STRING', 'search string'
|
47
|
+
def search(text)
|
48
|
+
puts "searching: #{text}"
|
49
|
+
results = @@client.cmd(:search, text: text, all: options[:all], rows: options[:rows])
|
50
|
+
if results.empty?
|
51
|
+
puts "find no result"
|
52
|
+
else
|
53
|
+
puts results.map{|v| v.join(":")}
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def self.run!
|
58
|
+
@@client = Librr::CmdClient.new('localhost', Settings.runner_port)
|
59
|
+
self.start(ARGV)
|
60
|
+
end
|
61
|
+
|
62
|
+
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
require 'eventmachine'
|
2
|
+
require 'evma_httpserver'
|
3
|
+
|
4
|
+
require 'json'
|
5
|
+
require 'rack'
|
6
|
+
|
7
|
+
|
8
|
+
class Librr::CmdServer
|
9
|
+
attr_accessor :monitor, :indexer
|
10
|
+
|
11
|
+
def init opts
|
12
|
+
self.monitor = opts[:monitor]
|
13
|
+
self.indexer = opts[:indexer]
|
14
|
+
CmdServerHandler.set_server(self)
|
15
|
+
end
|
16
|
+
|
17
|
+
def start(&block)
|
18
|
+
EventMachine.start_server "localhost", Settings.runner_port, CmdServerHandler
|
19
|
+
EM.add_timer(1){ block.call if block }
|
20
|
+
end
|
21
|
+
|
22
|
+
|
23
|
+
class CmdServerHandler < EM::Connection
|
24
|
+
include EM::HttpServer
|
25
|
+
include Librr::Logger::ClassLogger
|
26
|
+
|
27
|
+
def self.set_server(server)
|
28
|
+
@@server = server
|
29
|
+
end
|
30
|
+
|
31
|
+
def process_http_request
|
32
|
+
# puts @http_request_uri
|
33
|
+
response = EM::DelegatedHttpResponse.new(self)
|
34
|
+
response.status = 200
|
35
|
+
response.content_type 'application/json'
|
36
|
+
params = Rack::Utils.parse_nested_query(@http_post_content)
|
37
|
+
response.content = JSON.dump(self.handle_cmd(params))
|
38
|
+
response.send_response
|
39
|
+
end
|
40
|
+
|
41
|
+
def handle_cmd(params)
|
42
|
+
self.info "on receive: #{params.to_s}"
|
43
|
+
case params['cmd']
|
44
|
+
when 'ping'
|
45
|
+
'pong'
|
46
|
+
|
47
|
+
when 'stop'
|
48
|
+
puts "server stopping.."
|
49
|
+
EM.next_tick{
|
50
|
+
EM.stop
|
51
|
+
}
|
52
|
+
|
53
|
+
when 'add'
|
54
|
+
EM.next_tick{
|
55
|
+
@@server.monitor.add_directory(params['dir'])
|
56
|
+
}
|
57
|
+
|
58
|
+
when 'remove'
|
59
|
+
EM.next_tick{
|
60
|
+
@@server.monitor.remove_directory(params['dir'])
|
61
|
+
}
|
62
|
+
|
63
|
+
when 'list'
|
64
|
+
@@server.monitor.dirs.to_a
|
65
|
+
|
66
|
+
when 'reindex'
|
67
|
+
EM.next_tick{
|
68
|
+
@@server.monitor.reindex
|
69
|
+
}
|
70
|
+
|
71
|
+
when 'search'
|
72
|
+
@@server.indexer.search(params['text'], rows: params['rows'], all: params['all'])
|
73
|
+
|
74
|
+
else
|
75
|
+
raise Exception, "cmd unknown: #{params['cmd']}"
|
76
|
+
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
end
|
81
|
+
|
82
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'librr/settings'
|
2
|
+
|
3
|
+
module Configer
|
4
|
+
|
5
|
+
extend self
|
6
|
+
|
7
|
+
FILENAME = 'dir.conf'
|
8
|
+
|
9
|
+
def check_config_dir
|
10
|
+
conf_path = Settings::CONFIG_PATH
|
11
|
+
FileUtils.mkpath(conf_path) unless File.directory?(conf_path)
|
12
|
+
conf_path
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.load_dir_config
|
16
|
+
conf_path = self.check_config_dir
|
17
|
+
|
18
|
+
dc_file = Settings.in_dir(FILENAME)
|
19
|
+
if File.exists?(dc_file)
|
20
|
+
Set.new(File.read(dc_file).split("\n")).delete("")
|
21
|
+
else
|
22
|
+
[]
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.save_dir_config(config)
|
27
|
+
conf_path = self.check_config_dir
|
28
|
+
|
29
|
+
dc_file = Settings.in_dir(FILENAME)
|
30
|
+
File.write(dc_file, config.to_a.join("\n"))
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
class DelayIterator
|
2
|
+
def initialize(iter)
|
3
|
+
@iter = iter
|
4
|
+
end
|
5
|
+
|
6
|
+
def each(proc, finished=nil)
|
7
|
+
do_work = proc {
|
8
|
+
begin
|
9
|
+
item = @iter.next
|
10
|
+
proc.call(item)
|
11
|
+
EM.next_tick(&do_work)
|
12
|
+
rescue StopIteration
|
13
|
+
finished.call if finished
|
14
|
+
end
|
15
|
+
}
|
16
|
+
EM.next_tick(&do_work)
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,116 @@
|
|
1
|
+
require 'eventmachine'
|
2
|
+
require 'rb-fsevent'
|
3
|
+
require 'set'
|
4
|
+
|
5
|
+
require 'librr/logger'
|
6
|
+
require 'librr/configer'
|
7
|
+
|
8
|
+
|
9
|
+
class Librr::DirMonitor
|
10
|
+
include Librr::Logger::ClassLogger
|
11
|
+
|
12
|
+
attr_accessor :indexer, :dirs
|
13
|
+
|
14
|
+
def init opts
|
15
|
+
@pipe = nil
|
16
|
+
@new_start = false
|
17
|
+
@indexer = opts[:indexer]
|
18
|
+
|
19
|
+
self.dirs = Configer.load_dir_config
|
20
|
+
self.info "init dirs: #{self.dirs.to_a.to_s}"
|
21
|
+
end
|
22
|
+
|
23
|
+
def reindex
|
24
|
+
self.info "reindex"
|
25
|
+
@indexer.cleanup
|
26
|
+
self.dirs.each do |dir|
|
27
|
+
@indexer.index_directory(dir)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def add_directory(dir)
|
32
|
+
self.info "add dir: #{dir}"
|
33
|
+
@indexer.index_directory(dir)
|
34
|
+
self.dirs.add(dir)
|
35
|
+
Configer.save_dir_config(self.dirs)
|
36
|
+
self.info "save dir: #{self.dirs.to_a.to_s}"
|
37
|
+
self.start
|
38
|
+
end
|
39
|
+
|
40
|
+
def remove_directory(dir)
|
41
|
+
self.info "remove dir: #{dir}"
|
42
|
+
@indexer.remove_index_directory(dir)
|
43
|
+
self.dirs.delete(dir)
|
44
|
+
Configer.save_dir_config(self.dirs)
|
45
|
+
self.start
|
46
|
+
end
|
47
|
+
|
48
|
+
def post_init
|
49
|
+
@after_block.call if @after_block
|
50
|
+
end
|
51
|
+
|
52
|
+
def after_process_stop
|
53
|
+
self.start_process if @new_start
|
54
|
+
@new_start = false
|
55
|
+
end
|
56
|
+
|
57
|
+
def start &after_block
|
58
|
+
@after_block = after_block
|
59
|
+
|
60
|
+
if self.dirs.empty?
|
61
|
+
self.info "DIR empty, not start process."
|
62
|
+
self.post_init
|
63
|
+
return
|
64
|
+
end
|
65
|
+
|
66
|
+
if @pipe
|
67
|
+
@new_start = true
|
68
|
+
@pipe.close_connection
|
69
|
+
else
|
70
|
+
self.start_process
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def self.pid_file
|
75
|
+
Settings.in_dir('dir_watcher.pid')
|
76
|
+
end
|
77
|
+
|
78
|
+
def start_process
|
79
|
+
kill_process_by_file(self.class.pid_file)
|
80
|
+
|
81
|
+
cmd = [FSEvent.watcher_path] + ["--file-events"] + self.dirs.to_a
|
82
|
+
self.info "start process: #{cmd}"
|
83
|
+
@pipe = EM.popen(cmd, DirWatcher, self)
|
84
|
+
# TODO: write pid file
|
85
|
+
end
|
86
|
+
|
87
|
+
class DirWatcher < EventMachine::Connection
|
88
|
+
include Librr::Logger::ClassLogger
|
89
|
+
|
90
|
+
def initialize(monitor)
|
91
|
+
super
|
92
|
+
@monitor = monitor
|
93
|
+
end
|
94
|
+
|
95
|
+
def post_init
|
96
|
+
@monitor.post_init
|
97
|
+
end
|
98
|
+
|
99
|
+
def receive_data data
|
100
|
+
self.info "on receive data: #{data}"
|
101
|
+
changes = data.strip.split(':').map(&:strip).reject{|s| s == ''}
|
102
|
+
changes.each do |file|
|
103
|
+
@monitor.indexer.index_file(file)
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def unbind
|
108
|
+
self.info "dir monitor process stopped."
|
109
|
+
@monitor.after_process_stop
|
110
|
+
File.delete Librr::DirMonitor.pid_file rescue nil
|
111
|
+
end
|
112
|
+
|
113
|
+
end
|
114
|
+
|
115
|
+
end
|
116
|
+
|
@@ -0,0 +1,180 @@
|
|
1
|
+
require 'eventmachine'
|
2
|
+
# require 'rsolr-async' rescue nil
|
3
|
+
require 'rsolr'
|
4
|
+
|
5
|
+
|
6
|
+
require 'librr/lib'
|
7
|
+
require 'librr/settings'
|
8
|
+
require 'librr/delay_iterator'
|
9
|
+
|
10
|
+
|
11
|
+
class Librr::Indexer
|
12
|
+
include Librr::Logger::ClassLogger
|
13
|
+
|
14
|
+
attr_accessor :solr_started
|
15
|
+
|
16
|
+
SLICE_NUM = 300
|
17
|
+
|
18
|
+
def self.pid_file
|
19
|
+
Settings.in_dir('solr.pid')
|
20
|
+
end
|
21
|
+
|
22
|
+
def start &after_block
|
23
|
+
@after_block = after_block
|
24
|
+
|
25
|
+
kill_process_by_file(self.class.pid_file)
|
26
|
+
|
27
|
+
Dir.chdir File.join(Dir.pwd, 'solr') do
|
28
|
+
solr = 'java -jar start.jar'
|
29
|
+
solr_in, solr_out, solr_err = redirect_std do
|
30
|
+
EM.popen(solr, SolrManager)
|
31
|
+
# TODO: write pid file
|
32
|
+
end
|
33
|
+
EM.attach(solr_err, SolrOutHandler, self)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
module SolrManager
|
38
|
+
include Librr::Logger::ClassLogger
|
39
|
+
|
40
|
+
def post_init
|
41
|
+
self.info "start solr"
|
42
|
+
end
|
43
|
+
|
44
|
+
def receive_data data
|
45
|
+
self.info "receiving solr: #{data}"
|
46
|
+
end
|
47
|
+
|
48
|
+
def unbind
|
49
|
+
self.info "stop solr"
|
50
|
+
File.delete Librr::Indexer.pid_file rescue nil
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|
54
|
+
|
55
|
+
|
56
|
+
class SolrOutHandler < EventMachine::Connection
|
57
|
+
|
58
|
+
def initialize(indexer)
|
59
|
+
@indexer = indexer
|
60
|
+
end
|
61
|
+
|
62
|
+
def receive_data(data)
|
63
|
+
# File.open(Settings.in_dir('solr.log'), 'a+'){|f| f.write(data)}
|
64
|
+
if not @indexer.solr_started and data =~ /Started SocketConnector/
|
65
|
+
@indexer.after_start
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
70
|
+
|
71
|
+
|
72
|
+
def after_start
|
73
|
+
@solr_started = true
|
74
|
+
self.info 'after solr start'
|
75
|
+
|
76
|
+
@solr = RSolr.connect(
|
77
|
+
url: "http://localhost:#{Settings.solr_port}/solr",
|
78
|
+
read_timeout: 10, open_timeout: 10)
|
79
|
+
@after_block.call if @after_block
|
80
|
+
end
|
81
|
+
|
82
|
+
def run_solr &block
|
83
|
+
retry_times = 2
|
84
|
+
begin
|
85
|
+
block.call
|
86
|
+
rescue Net::ReadTimeout
|
87
|
+
retry_times -= 1
|
88
|
+
retry if retry_times >= 0
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
def cleanup
|
93
|
+
self.info 'cleanup'
|
94
|
+
self.run_solr {
|
95
|
+
@solr.delete_by_query '*:*'
|
96
|
+
@solr.commit
|
97
|
+
}
|
98
|
+
end
|
99
|
+
|
100
|
+
def index_directory(dir)
|
101
|
+
self.info "index dir: #{dir}"
|
102
|
+
files = Dir.glob(File.join(dir, "**/*"))
|
103
|
+
EM::Iterator.new(files)
|
104
|
+
.each(
|
105
|
+
proc { |file, iter|
|
106
|
+
if File.file?(file)
|
107
|
+
self.index_file(file){ iter.next }
|
108
|
+
else
|
109
|
+
iter.next
|
110
|
+
end
|
111
|
+
},
|
112
|
+
proc { self.info "index dir finished: #{dir}" }
|
113
|
+
)
|
114
|
+
end
|
115
|
+
|
116
|
+
def remove_index_directory(dir)
|
117
|
+
self.info "remove dir: #{dir}"
|
118
|
+
self.run_solr {
|
119
|
+
@solr.delete_by_query "filename:#{dir}*"
|
120
|
+
@solr.commit
|
121
|
+
}
|
122
|
+
end
|
123
|
+
|
124
|
+
def index_file(file, &block)
|
125
|
+
return if File.basename(file) =~ Settings.escape_files
|
126
|
+
|
127
|
+
self.run_solr {
|
128
|
+
@solr.delete_by_query "filename:#{file}"
|
129
|
+
@solr.commit
|
130
|
+
}
|
131
|
+
|
132
|
+
unless File.exists?(file)
|
133
|
+
self.info "remove index file: #{file}"
|
134
|
+
block.call if block
|
135
|
+
return
|
136
|
+
end
|
137
|
+
|
138
|
+
self.info "index file: #{file}"
|
139
|
+
f = File.open(file)
|
140
|
+
enum = f.each.each_slice(SLICE_NUM).each_with_index
|
141
|
+
self.info "file indexing...."
|
142
|
+
DelayIterator.new(enum)
|
143
|
+
.each(
|
144
|
+
proc { |lines, i|
|
145
|
+
data = lines.each_with_index.map do |line, j|
|
146
|
+
num = SLICE_NUM * i + j
|
147
|
+
line = fix_encoding(line).rstrip
|
148
|
+
{id: SecureRandom.uuid, filename: file, linenum: num, line: line}
|
149
|
+
end
|
150
|
+
|
151
|
+
self.run_solr {
|
152
|
+
@solr.add data
|
153
|
+
@solr.commit
|
154
|
+
}
|
155
|
+
|
156
|
+
self.info "working on lines: #{i*SLICE_NUM}"
|
157
|
+
},
|
158
|
+
proc {
|
159
|
+
f.close
|
160
|
+
block.call if block
|
161
|
+
}
|
162
|
+
)
|
163
|
+
end
|
164
|
+
|
165
|
+
def search(str, opts={})
|
166
|
+
self.info "search: #{str}"
|
167
|
+
|
168
|
+
rows = opts[:rows] || 30
|
169
|
+
rows = (2 ** 31 - 1) if opts[:all]
|
170
|
+
|
171
|
+
result = self.run_solr {
|
172
|
+
@solr.get 'select', params: {q: "line:#{str}", rows: rows}
|
173
|
+
}
|
174
|
+
|
175
|
+
result['response']['docs'].map do |row|
|
176
|
+
[row['filename'], row['linenum'], row['line']].flatten
|
177
|
+
end
|
178
|
+
end
|
179
|
+
|
180
|
+
end
|
data/lib/librr/lib.rb
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
def redirect_std
|
2
|
+
stdin = $stdin.dup
|
3
|
+
stdout = $stdout.dup
|
4
|
+
stderr = $stderr.dup
|
5
|
+
|
6
|
+
ri, wi = IO::pipe
|
7
|
+
ro, wo = IO::pipe
|
8
|
+
re, we = IO::pipe
|
9
|
+
|
10
|
+
$stdin.reopen ri
|
11
|
+
$stdout.reopen wo
|
12
|
+
$stderr.reopen we
|
13
|
+
|
14
|
+
yield
|
15
|
+
|
16
|
+
$stdin.reopen stdin
|
17
|
+
$stdout.reopen stdout
|
18
|
+
$stderr.reopen stderr
|
19
|
+
[wi, ro, re]
|
20
|
+
end
|
21
|
+
|
22
|
+
|
23
|
+
def fix_encoding text
|
24
|
+
# solution copy from:
|
25
|
+
# http://stackoverflow.com/questions/11375342/stringencode-not-fixing-invalid-byte-sequence-in-utf-8-error
|
26
|
+
text
|
27
|
+
.encode('UTF-16', undef: :replace, invalid: :replace, replace: "")
|
28
|
+
.encode('UTF-8')
|
29
|
+
end
|
30
|
+
|
31
|
+
def kill_process_by_file file
|
32
|
+
begin
|
33
|
+
pid = File.read(file).to_i
|
34
|
+
Process.kill 'TERM', pid if pid > 0
|
35
|
+
File.delete file
|
36
|
+
rescue
|
37
|
+
end
|
38
|
+
end
|
data/lib/librr/logger.rb
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'logger'
|
2
|
+
|
3
|
+
class Librr::Logger
|
4
|
+
|
5
|
+
module ClassLogger
|
6
|
+
def info(text)
|
7
|
+
$logger.info(self.class.name){ text }
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.create_logger
|
12
|
+
logger = Logger.new(STDOUT)
|
13
|
+
# logger.level = Logger::WARN
|
14
|
+
logger.level = Logger::DEBUG
|
15
|
+
logger
|
16
|
+
end
|
17
|
+
|
18
|
+
end
|
19
|
+
|
20
|
+
$logger ||= Librr::Logger.create_logger
|
21
|
+
|
data/lib/librr/runner.rb
ADDED
@@ -0,0 +1,56 @@
|
|
1
|
+
require 'eventmachine'
|
2
|
+
|
3
|
+
require 'librr/settings'
|
4
|
+
require 'librr/indexer'
|
5
|
+
require 'librr/dir_monitor'
|
6
|
+
require 'librr/cmd_server'
|
7
|
+
|
8
|
+
|
9
|
+
EventMachine.kqueue = true if EventMachine.kqueue?
|
10
|
+
|
11
|
+
|
12
|
+
class Librr::Runner
|
13
|
+
def run!
|
14
|
+
self.clear_pid
|
15
|
+
@stoping = false
|
16
|
+
|
17
|
+
EventMachine.run do
|
18
|
+
trap("SIGINT") do
|
19
|
+
return if @stoping
|
20
|
+
@stoping = true
|
21
|
+
|
22
|
+
EM.stop
|
23
|
+
puts "eventmachine graceful stops."
|
24
|
+
# todo commandline still show ^C?
|
25
|
+
self.clear_pid
|
26
|
+
end
|
27
|
+
|
28
|
+
indexer = Librr::Indexer.new
|
29
|
+
monitor = Librr::DirMonitor.new
|
30
|
+
server = Librr::CmdServer.new
|
31
|
+
|
32
|
+
monitor.init(indexer: indexer)
|
33
|
+
server.init(indexer: indexer, monitor: monitor)
|
34
|
+
|
35
|
+
indexer.start do
|
36
|
+
monitor.start do
|
37
|
+
server.start do
|
38
|
+
puts "server started"
|
39
|
+
self.write_pid
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def write_pid
|
48
|
+
filename = Settings::PID_FILE
|
49
|
+
File.open(filename, 'w+'){ |f| f.write(Process.pid.to_s) }
|
50
|
+
end
|
51
|
+
|
52
|
+
def clear_pid
|
53
|
+
filename = Settings::PID_FILE
|
54
|
+
File.delete(filename) rescue nil
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
# copy from gem daemons file: daemonize.rb
|
2
|
+
# todo: may has secruity risk
|
3
|
+
require 'librr/lib'
|
4
|
+
|
5
|
+
module ServerStarter
|
6
|
+
extend self
|
7
|
+
|
8
|
+
def run
|
9
|
+
require 'librr'
|
10
|
+
require 'librr/runner'
|
11
|
+
|
12
|
+
Librr::Runner.new.run!
|
13
|
+
end
|
14
|
+
|
15
|
+
def start_server(sync)
|
16
|
+
puts 'server starting..'
|
17
|
+
return self.run if sync
|
18
|
+
|
19
|
+
Process.fork do
|
20
|
+
sess_id = Process.setsid
|
21
|
+
Process.fork do
|
22
|
+
redirect_std do
|
23
|
+
$logger.info "daemon started."
|
24
|
+
self.run
|
25
|
+
end
|
26
|
+
exit
|
27
|
+
end
|
28
|
+
exit
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def wait_for_server_started &block
|
33
|
+
5.times.each do
|
34
|
+
sleep(2)
|
35
|
+
puts 'waiting for server starting..'
|
36
|
+
|
37
|
+
if File.exists?(Settings::PID_FILE)
|
38
|
+
return block.call if block
|
39
|
+
end
|
40
|
+
end
|
41
|
+
puts "server not starting, something is wrong."
|
42
|
+
exit
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
|
3
|
+
class Settings
|
4
|
+
|
5
|
+
CONFIG_PATH = File.expand_path('~/.librr/')
|
6
|
+
|
7
|
+
def self.in_dir(filename)
|
8
|
+
File.join(CONFIG_PATH, filename)
|
9
|
+
end
|
10
|
+
|
11
|
+
|
12
|
+
CONFIG_FILE = self.in_dir('config')
|
13
|
+
PID_FILE = self.in_dir('server.pid')
|
14
|
+
|
15
|
+
DEFAULTS = {
|
16
|
+
runner_port: 4512,
|
17
|
+
config_path: CONFIG_PATH,
|
18
|
+
escape_files: /[#~]$|^[\.#]/,
|
19
|
+
solr_port: 8901,
|
20
|
+
}
|
21
|
+
|
22
|
+
def self.reload
|
23
|
+
if File.exists?(CONFIG_FILE)
|
24
|
+
begin
|
25
|
+
@@config = YAML.load File.read(CONFIG_FILE)
|
26
|
+
raise unless @@config.kind_of?(Hash)
|
27
|
+
rescue
|
28
|
+
raise "config file format error: #{CONFIG_FILE}"
|
29
|
+
end
|
30
|
+
else
|
31
|
+
@@config = DEFAULTS.dup
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.method_missing(name)
|
36
|
+
self.reload unless defined?(@@config)
|
37
|
+
|
38
|
+
if @@config.include?(name)
|
39
|
+
@@config[name]
|
40
|
+
else
|
41
|
+
super
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
data/lib/librr.rb
ADDED
data/solr/makefile
ADDED
data/solr/start.jar
ADDED
Binary file
|
metadata
ADDED
@@ -0,0 +1,196 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: librr
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- linjunhalida
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2013-11-29 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: thor
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0.8'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ~>
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0.8'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: eventmachine
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: eventmachine_httpserver_update
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - '>='
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rsolr-async
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - '>='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: rb-fsevent
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - '>='
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - '>='
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: rack
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - '>='
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :runtime
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - '>='
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: rake
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - '>='
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - '>='
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: sass
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - '>='
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - '>='
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: rspec
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - '>='
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '0'
|
132
|
+
type: :development
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - '>='
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '0'
|
139
|
+
description: |2
|
140
|
+
It is a tool to to index & search your text based documentation system.
|
141
|
+
It use solr for fulltext index.
|
142
|
+
email: linjunhalida@gmail.com
|
143
|
+
executables:
|
144
|
+
- librr
|
145
|
+
extensions: []
|
146
|
+
extra_rdoc_files:
|
147
|
+
- README.md
|
148
|
+
- LICENSE
|
149
|
+
files:
|
150
|
+
- README.md
|
151
|
+
- LICENSE
|
152
|
+
- bin/daemon_run.rb~
|
153
|
+
- bin/librr
|
154
|
+
- bin/librr~
|
155
|
+
- bin/test~
|
156
|
+
- lib/librr/cmd_client.rb
|
157
|
+
- lib/librr/cmd_parser.rb
|
158
|
+
- lib/librr/cmd_server.rb
|
159
|
+
- lib/librr/configer.rb
|
160
|
+
- lib/librr/delay_iterator.rb
|
161
|
+
- lib/librr/dir_monitor.rb
|
162
|
+
- lib/librr/indexer.rb
|
163
|
+
- lib/librr/lib.rb
|
164
|
+
- lib/librr/logger.rb
|
165
|
+
- lib/librr/runner.rb
|
166
|
+
- lib/librr/server_starter.rb
|
167
|
+
- lib/librr/settings.rb
|
168
|
+
- lib/librr/version.rb
|
169
|
+
- lib/librr.rb
|
170
|
+
- solr/makefile
|
171
|
+
- solr/start.jar
|
172
|
+
homepage: https://github.com/halida/librr
|
173
|
+
licenses:
|
174
|
+
- MIT
|
175
|
+
metadata: {}
|
176
|
+
post_install_message:
|
177
|
+
rdoc_options: []
|
178
|
+
require_paths:
|
179
|
+
- lib
|
180
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
181
|
+
requirements:
|
182
|
+
- - '>='
|
183
|
+
- !ruby/object:Gem::Version
|
184
|
+
version: 1.9.3
|
185
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
186
|
+
requirements:
|
187
|
+
- - '>='
|
188
|
+
- !ruby/object:Gem::Version
|
189
|
+
version: '0'
|
190
|
+
requirements: []
|
191
|
+
rubyforge_project:
|
192
|
+
rubygems_version: 2.1.11
|
193
|
+
signing_key:
|
194
|
+
specification_version: 4
|
195
|
+
summary: line based personal documentation search system.
|
196
|
+
test_files: []
|