hopi 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +4 -0
- data/Gemfile.lock +7 -1
- data/lib/hopi/crawler.rb +22 -0
- data/lib/hopi/execeptions.rb +7 -0
- data/lib/hopi/tor_client.rb +53 -0
- data/lib/hopi/version.rb +1 -1
- data/lib/hopi.rb +4 -5
- metadata +3 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b4082f0fcf32faae1006aff6dbce911d50bc93e7148a4abbb0a8660a686ea0e6
|
4
|
+
data.tar.gz: 2ee71e2345d6e88afe074d01b10994dc88a3ba4cb33a4da5ed4fb1a40a410976
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e0d930413a3b601a8f53eb672b4211c00e32a8cca3cb0d97ad85aa78e0ccb6bb34ca37e9f1298ae9f934829413adecc535e1869810030c10a03fa5bc58ab0581
|
7
|
+
data.tar.gz: 9ebc3cefda69dc4183ed462ee0784319ad313e4a044d84815d09c0087462f3ce3c8ca32c8f1fcf0facda27cf2288ba2500cc95c5e61e7edbb6437b09372f1b09
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
hopi (0.1.
|
4
|
+
hopi (0.1.1)
|
5
5
|
|
6
6
|
GEM
|
7
7
|
remote: https://rubygems.org/
|
@@ -11,6 +11,7 @@ GEM
|
|
11
11
|
benchmark (0.2.0)
|
12
12
|
diff-lcs (1.5.0)
|
13
13
|
e2mmap (0.1.0)
|
14
|
+
ffi (1.15.5)
|
14
15
|
jaro_winkler (1.5.4)
|
15
16
|
kramdown (2.4.0)
|
16
17
|
rexml
|
@@ -53,6 +54,7 @@ GEM
|
|
53
54
|
rubocop-ast (1.18.0)
|
54
55
|
parser (>= 3.1.1.0)
|
55
56
|
ruby-progressbar (1.11.0)
|
57
|
+
socksify (1.7.1)
|
56
58
|
solargraph (0.45.0)
|
57
59
|
backport (~> 1.2)
|
58
60
|
benchmark
|
@@ -68,6 +70,8 @@ GEM
|
|
68
70
|
thor (~> 1.0)
|
69
71
|
tilt (~> 2.0)
|
70
72
|
yard (~> 0.9, >= 0.9.24)
|
73
|
+
sys-proctable (1.2.6)
|
74
|
+
ffi
|
71
75
|
thor (1.2.1)
|
72
76
|
tilt (2.0.10)
|
73
77
|
unicode-display_width (2.1.0)
|
@@ -82,7 +86,9 @@ DEPENDENCIES
|
|
82
86
|
hopi!
|
83
87
|
rake (~> 13.0)
|
84
88
|
rspec (~> 3.0)
|
89
|
+
socksify (~> 1.7)
|
85
90
|
solargraph (~> 0.45.0)
|
91
|
+
sys-proctable (~> 1.2)
|
86
92
|
|
87
93
|
BUNDLED WITH
|
88
94
|
2.3.14
|
data/lib/hopi/crawler.rb
CHANGED
@@ -0,0 +1,22 @@
|
|
1
|
+
class Crawler
|
2
|
+
|
3
|
+
include Process
|
4
|
+
|
5
|
+
def initialize crawler_count = 1
|
6
|
+
@crawler_count = crawler_count
|
7
|
+
end
|
8
|
+
|
9
|
+
def pooling_crawl crawl_target
|
10
|
+
|
11
|
+
pool_size = crawl_target.count / @crawler_count
|
12
|
+
pooled_crawl_target = crawl_target.each_slice pool_size
|
13
|
+
|
14
|
+
pooled_crawl_target.to_a.map do |pool|
|
15
|
+
Thread.new do
|
16
|
+
pool.map do |pool_unit|
|
17
|
+
yield pool_unit
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end.map(&:join).flatten
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
class TorClient
|
2
|
+
|
3
|
+
include Process
|
4
|
+
|
5
|
+
attr_reader :tor_pid, :tor_available, :tor_master
|
6
|
+
|
7
|
+
def initialize torrc_path=nil
|
8
|
+
@torrc_path = torrc_path
|
9
|
+
@torr_log_output = []
|
10
|
+
end
|
11
|
+
|
12
|
+
def tor_available?
|
13
|
+
find_executable 'tor'
|
14
|
+
end
|
15
|
+
|
16
|
+
def down
|
17
|
+
@tor_master.close
|
18
|
+
kill :QUIT, @tor_pid
|
19
|
+
puts "🔻Killed Tor instance"
|
20
|
+
true
|
21
|
+
end
|
22
|
+
|
23
|
+
def up
|
24
|
+
raise FailedToSpawnTor "[err] Tor executable is unavailable. Consider modifying your PATH env" unless tor_available?
|
25
|
+
|
26
|
+
master, slave = PTY.open
|
27
|
+
read,write = IO.pipe
|
28
|
+
|
29
|
+
if @torrc_path.nil?
|
30
|
+
pid = spawn "tor", :in=>read, :out=>slave
|
31
|
+
else
|
32
|
+
pid = spawn "tor -f #{torrc_path}", :in=>read, :out=>slave
|
33
|
+
end
|
34
|
+
|
35
|
+
read.close
|
36
|
+
slave.close
|
37
|
+
write.close
|
38
|
+
|
39
|
+
puts "🤞Waiting for tor to spawn ..."
|
40
|
+
|
41
|
+
loop do
|
42
|
+
line = master.gets
|
43
|
+
raise FailedToSpawnTor.new "[err] Failed to spawn tor process" if line.nil?
|
44
|
+
raise FailedToSpawnTor.new line if ["[warn]", "[err]"].any?{|loglevel| line.include? loglevel}
|
45
|
+
if line.include? "[notice] Bootstrapped 100% (done)"
|
46
|
+
@tor_pid = pid
|
47
|
+
@tor_master = master
|
48
|
+
puts "🚀 Spawned tor process"
|
49
|
+
return @tor_pid
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
data/lib/hopi/version.rb
CHANGED
data/lib/hopi.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: hopi
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Trance
|
@@ -28,6 +28,8 @@ files:
|
|
28
28
|
- hopi.gemspec
|
29
29
|
- lib/hopi.rb
|
30
30
|
- lib/hopi/crawler.rb
|
31
|
+
- lib/hopi/execeptions.rb
|
32
|
+
- lib/hopi/tor_client.rb
|
31
33
|
- lib/hopi/version.rb
|
32
34
|
- sig/hopi.rbs
|
33
35
|
homepage: https://github.com/0xTrance/hopi/blob/main/README.md
|