hopi 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 15296742ae4fb99da5a3fffeb0a19fe94bc8407986b0698f83998b188affdc1c
4
- data.tar.gz: 722064e20e8b4bc5d5e18516b1966a576197696882435d42063b3a1bfc442eee
3
+ metadata.gz: b4082f0fcf32faae1006aff6dbce911d50bc93e7148a4abbb0a8660a686ea0e6
4
+ data.tar.gz: 2ee71e2345d6e88afe074d01b10994dc88a3ba4cb33a4da5ed4fb1a40a410976
5
5
  SHA512:
6
- metadata.gz: 7f8cf70d5d7d60da500c49129a50d103873e7f40153989fa0d80fcea860b98f1f39bbb0f57a5fc146c449e6b407e50f39994fd07a58a77d7ce2c73262c2b868f
7
- data.tar.gz: 5e936b583d787a630e807e1b7a86eb28a8cda2a938a9919b6f1c5751e322bbd2c6b1b07e47b6af82b5c1c76a4078efdf7e43590a3b00717edf562f60dbfaa60e
6
+ metadata.gz: e0d930413a3b601a8f53eb672b4211c00e32a8cca3cb0d97ad85aa78e0ccb6bb34ca37e9f1298ae9f934829413adecc535e1869810030c10a03fa5bc58ab0581
7
+ data.tar.gz: 9ebc3cefda69dc4183ed462ee0784319ad313e4a044d84815d09c0087462f3ce3c8ca32c8f1fcf0facda27cf2288ba2500cc95c5e61e7edbb6437b09372f1b09
data/Gemfile CHANGED
@@ -10,3 +10,7 @@ gem "rake", "~> 13.0"
10
10
  gem "rspec", "~> 3.0"
11
11
 
12
12
  gem "solargraph", "~> 0.45.0", :group => :development
13
+
14
+ gem "socksify", "~> 1.7"
15
+
16
+ gem "sys-proctable", "~> 1.2"
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- hopi (0.1.0)
4
+ hopi (0.1.1)
5
5
 
6
6
  GEM
7
7
  remote: https://rubygems.org/
@@ -11,6 +11,7 @@ GEM
11
11
  benchmark (0.2.0)
12
12
  diff-lcs (1.5.0)
13
13
  e2mmap (0.1.0)
14
+ ffi (1.15.5)
14
15
  jaro_winkler (1.5.4)
15
16
  kramdown (2.4.0)
16
17
  rexml
@@ -53,6 +54,7 @@ GEM
53
54
  rubocop-ast (1.18.0)
54
55
  parser (>= 3.1.1.0)
55
56
  ruby-progressbar (1.11.0)
57
+ socksify (1.7.1)
56
58
  solargraph (0.45.0)
57
59
  backport (~> 1.2)
58
60
  benchmark
@@ -68,6 +70,8 @@ GEM
68
70
  thor (~> 1.0)
69
71
  tilt (~> 2.0)
70
72
  yard (~> 0.9, >= 0.9.24)
73
+ sys-proctable (1.2.6)
74
+ ffi
71
75
  thor (1.2.1)
72
76
  tilt (2.0.10)
73
77
  unicode-display_width (2.1.0)
@@ -82,7 +86,9 @@ DEPENDENCIES
82
86
  hopi!
83
87
  rake (~> 13.0)
84
88
  rspec (~> 3.0)
89
+ socksify (~> 1.7)
85
90
  solargraph (~> 0.45.0)
91
+ sys-proctable (~> 1.2)
86
92
 
87
93
  BUNDLED WITH
88
94
  2.3.14
data/lib/hopi/crawler.rb CHANGED
@@ -0,0 +1,22 @@
1
+ class Crawler
2
+
3
+ include Process
4
+
5
+ def initialize crawler_count = 1
6
+ @crawler_count = crawler_count
7
+ end
8
+
9
+ def pooling_crawl crawl_target
10
+
11
+ pool_size = crawl_target.count / @crawler_count
12
+ pooled_crawl_target = crawl_target.each_slice pool_size
13
+
14
+ pooled_crawl_target.to_a.map do |pool|
15
+ Thread.new do
16
+ pool.map do |pool_unit|
17
+ yield pool_unit
18
+ end
19
+ end
20
+ end.map(&:join).flatten
21
+ end
22
+ end
@@ -0,0 +1,7 @@
1
+
2
+ # Tor executable is missing error
3
+ class FailedToSpawnTor < StandardError
4
+ def initialize msg
5
+ super "😔 #{msg}"
6
+ end
7
+ end
@@ -0,0 +1,53 @@
1
+ class TorClient
2
+
3
+ include Process
4
+
5
+ attr_reader :tor_pid, :tor_available, :tor_master
6
+
7
+ def initialize torrc_path=nil
8
+ @torrc_path = torrc_path
9
+ @torr_log_output = []
10
+ end
11
+
12
+ def tor_available?
13
+ find_executable 'tor'
14
+ end
15
+
16
+ def down
17
+ @tor_master.close
18
+ kill :QUIT, @tor_pid
19
+ puts "🔻Killed Tor instance"
20
+ true
21
+ end
22
+
23
+ def up
24
+ raise FailedToSpawnTor "[err] Tor executable is unavailable. Consider modifying your PATH env" unless tor_available?
25
+
26
+ master, slave = PTY.open
27
+ read,write = IO.pipe
28
+
29
+ if @torrc_path.nil?
30
+ pid = spawn "tor", :in=>read, :out=>slave
31
+ else
32
+ pid = spawn "tor -f #{torrc_path}", :in=>read, :out=>slave
33
+ end
34
+
35
+ read.close
36
+ slave.close
37
+ write.close
38
+
39
+ puts "🤞Waiting for tor to spawn ..."
40
+
41
+ loop do
42
+ line = master.gets
43
+ raise FailedToSpawnTor.new "[err] Failed to spawn tor process" if line.nil?
44
+ raise FailedToSpawnTor.new line if ["[warn]", "[err]"].any?{|loglevel| line.include? loglevel}
45
+ if line.include? "[notice] Bootstrapped 100% (done)"
46
+ @tor_pid = pid
47
+ @tor_master = master
48
+ puts "🚀 Spawned tor process"
49
+ return @tor_pid
50
+ end
51
+ end
52
+ end
53
+ end
data/lib/hopi/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Hopi
4
- VERSION = "0.1.0"
4
+ VERSION = "0.1.1"
5
5
  end
data/lib/hopi.rb CHANGED
@@ -1,8 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative "hopi/version"
4
-
5
- module Hopi
6
- class Error < StandardError; end
7
- # Your code goes here...
8
- end
4
+ require_relative 'hopi/execeptions'
5
+ require_relative "hopi/tor_client"
6
+ require "mkmf"
7
+ require "pty"
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: hopi
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.1.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Trance
@@ -28,6 +28,8 @@ files:
28
28
  - hopi.gemspec
29
29
  - lib/hopi.rb
30
30
  - lib/hopi/crawler.rb
31
+ - lib/hopi/execeptions.rb
32
+ - lib/hopi/tor_client.rb
31
33
  - lib/hopi/version.rb
32
34
  - sig/hopi.rbs
33
35
  homepage: https://github.com/0xTrance/hopi/blob/main/README.md