hopi 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 15296742ae4fb99da5a3fffeb0a19fe94bc8407986b0698f83998b188affdc1c
-  data.tar.gz: 722064e20e8b4bc5d5e18516b1966a576197696882435d42063b3a1bfc442eee
+  metadata.gz: b4082f0fcf32faae1006aff6dbce911d50bc93e7148a4abbb0a8660a686ea0e6
+  data.tar.gz: 2ee71e2345d6e88afe074d01b10994dc88a3ba4cb33a4da5ed4fb1a40a410976
 SHA512:
-  metadata.gz: 7f8cf70d5d7d60da500c49129a50d103873e7f40153989fa0d80fcea860b98f1f39bbb0f57a5fc146c449e6b407e50f39994fd07a58a77d7ce2c73262c2b868f
-  data.tar.gz: 5e936b583d787a630e807e1b7a86eb28a8cda2a938a9919b6f1c5751e322bbd2c6b1b07e47b6af82b5c1c76a4078efdf7e43590a3b00717edf562f60dbfaa60e
+  metadata.gz: e0d930413a3b601a8f53eb672b4211c00e32a8cca3cb0d97ad85aa78e0ccb6bb34ca37e9f1298ae9f934829413adecc535e1869810030c10a03fa5bc58ab0581
+  data.tar.gz: 9ebc3cefda69dc4183ed462ee0784319ad313e4a044d84815d09c0087462f3ce3c8ca32c8f1fcf0facda27cf2288ba2500cc95c5e61e7edbb6437b09372f1b09

data/Gemfile

@@ -10,3 +10,7 @@ gem "rake", "~> 13.0"
 gem "rspec", "~> 3.0"
 
 gem "solargraph", "~> 0.45.0", :group => :development
+
+gem "socksify", "~> 1.7"
+
+gem "sys-proctable", "~> 1.2"

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    hopi (0.1.0)
+    hopi (0.1.1)
 
 GEM
   remote: https://rubygems.org/
@@ -11,6 +11,7 @@ GEM
     benchmark (0.2.0)
     diff-lcs (1.5.0)
     e2mmap (0.1.0)
+    ffi (1.15.5)
     jaro_winkler (1.5.4)
     kramdown (2.4.0)
       rexml
@@ -53,6 +54,7 @@ GEM
     rubocop-ast (1.18.0)
       parser (>= 3.1.1.0)
     ruby-progressbar (1.11.0)
+    socksify (1.7.1)
     solargraph (0.45.0)
       backport (~> 1.2)
       benchmark
@@ -68,6 +70,8 @@ GEM
       thor (~> 1.0)
       tilt (~> 2.0)
       yard (~> 0.9, >= 0.9.24)
+    sys-proctable (1.2.6)
+      ffi
     thor (1.2.1)
     tilt (2.0.10)
     unicode-display_width (2.1.0)
@@ -82,7 +86,9 @@ DEPENDENCIES
   hopi!
   rake (~> 13.0)
   rspec (~> 3.0)
+  socksify (~> 1.7)
   solargraph (~> 0.45.0)
+  sys-proctable (~> 1.2)
 
 BUNDLED WITH
    2.3.14

data/lib/hopi/crawler.rb

@@ -0,0 +1,22 @@
+class Crawler
+
+  include Process
+
+  def initialize crawler_count = 1
+    @crawler_count = crawler_count
+  end
+
+  def pooling_crawl crawl_target
+
+    pool_size = crawl_target.count / @crawler_count
+    pooled_crawl_target = crawl_target.each_slice pool_size
+
+    pooled_crawl_target.to_a.map do |pool|
+      Thread.new do
+        pool.map do |pool_unit|
+          yield pool_unit
+        end
+      end
+    end.map(&:join).flatten
+  end
+end

data/lib/hopi/execeptions.rb

@@ -0,0 +1,7 @@
+
+# Tor executable is missing error
+class FailedToSpawnTor < StandardError
+  def initialize msg
+    super "😔 #{msg}"
+  end
+end

data/lib/hopi/tor_client.rb

@@ -0,0 +1,53 @@
+class TorClient
+
+  include Process
+
+  attr_reader :tor_pid, :tor_available, :tor_master
+
+  def initialize torrc_path=nil
+    @torrc_path = torrc_path
+    @torr_log_output = []
+  end
+
+  def tor_available?
+    find_executable 'tor'
+  end
+
+  def down
+    @tor_master.close
+    kill :QUIT, @tor_pid
+    puts "🔻Killed Tor instance"
+    true
+  end
+
+  def up
+    raise FailedToSpawnTor "[err] Tor executable is unavailable. Consider modifying your PATH env" unless tor_available?
+
+    master, slave = PTY.open
+    read,write = IO.pipe
+
+    if @torrc_path.nil?
+      pid = spawn "tor", :in=>read, :out=>slave
+    else
+      pid = spawn "tor -f #{torrc_path}", :in=>read, :out=>slave
+    end
+
+    read.close
+    slave.close
+    write.close
+      
+    puts "🤞Waiting for tor to spawn ..."
+
+    loop do
+      line = master.gets
+      raise FailedToSpawnTor.new "[err] Failed to spawn tor process" if line.nil?
+      raise FailedToSpawnTor.new line if ["[warn]", "[err]"].any?{|loglevel| line.include? loglevel}
+      if line.include? "[notice] Bootstrapped 100% (done)"
+        @tor_pid = pid
+        @tor_master = master
+        puts "🚀 Spawned tor process"
+        return @tor_pid
+      end
+    end
+  end
+end

data/lib/hopi/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Hopi
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/lib/hopi.rb

@@ -1,8 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "hopi/version"
-
-module Hopi
-  class Error < StandardError; end
-  # Your code goes here...
-end
+require_relative 'hopi/execeptions'
+require_relative "hopi/tor_client"
+require "mkmf"
+require "pty"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: hopi
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Trance
@@ -28,6 +28,8 @@ files:
 - hopi.gemspec
 - lib/hopi.rb
 - lib/hopi/crawler.rb
+- lib/hopi/execeptions.rb
+- lib/hopi/tor_client.rb
 - lib/hopi/version.rb
 - sig/hopi.rbs
 homepage: https://github.com/0xTrance/hopi/blob/main/README.md