sengi 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 027ebc7d9b5e1a8f6d3bb4b5c434beb27a152c24
+  data.tar.gz: c79a8f8dd0fdf168071b0ce93049f93e9fcb9422
+SHA512:
+  metadata.gz: 3799cf4daca51694b6f375d7ed68db4111d21c95176860845a5970e25ee8d3d056250a45f630776d4922b9d835b93f4134fa7facc2c5f5a4809a4cca568f022a
+  data.tar.gz: 9fabd1047750167bb3e7daaa7638fcd013a3fb04ede390b8b845d7743190a97bc81819f0af59938da8d92ef4d7bb5e501f1c5642b5361d5a60da0d55fd183b31

data/.gitignore

@@ -0,0 +1,7 @@
+/.bundle/
+.setup
+*.rdb
+/bin/dev
+/run/
+/tmp/
+README.html

data/Gemfile

@@ -0,0 +1,3 @@
+
+source 'https://rubygems.org'
+gemspec

data/Gemfile.lock

@@ -0,0 +1,71 @@
+PATH
+  remote: .
+  specs:
+    sengi (0.1.0.pre.dev.6)
+      activesupport (~> 4.2)
+      cookiejar (~> 0.3)
+      hiredis (~> 0.6)
+      nokogiri (~> 1.6)
+      redis (~> 3.2)
+      resque (~> 1.26)
+      resque-scheduler (~> 4.1)
+      thefox-ext (~> 1.4)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.2.6)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    cookiejar (0.3.3)
+    hiredis (0.6.1)
+    i18n (0.7.0)
+    json (1.8.3)
+    mini_portile2 (2.0.0)
+    minitest (5.8.4)
+    mono_logger (1.1.0)
+    multi_json (1.11.2)
+    nokogiri (1.6.7.2)
+      mini_portile2 (~> 2.0.0.rc2)
+    rack (1.6.4)
+    rack-protection (1.5.3)
+      rack
+    redis (3.3.0)
+    redis-namespace (1.5.2)
+      redis (~> 3.0, >= 3.0.4)
+    resque (1.26.0)
+      mono_logger (~> 1.0)
+      multi_json (~> 1.0)
+      redis-namespace (~> 1.3)
+      sinatra (>= 0.9.2)
+      vegas (~> 0.1.2)
+    resque-scheduler (4.1.0)
+      mono_logger (~> 1.0)
+      redis (~> 3.0)
+      resque (~> 1.25)
+      rufus-scheduler (~> 3.0)
+    rufus-scheduler (3.2.0)
+    sinatra (1.4.7)
+      rack (~> 1.5)
+      rack-protection (~> 1.4)
+      tilt (>= 1.3, < 3)
+    thefox-ext (1.4.1)
+    thread_safe (0.3.5)
+    tilt (2.0.2)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+    vegas (0.1.11)
+      rack (>= 1.0.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  minitest (~> 5.8)
+  sengi!
+
+BUNDLED WITH
+   1.11.2

data/Makefile

@@ -0,0 +1,23 @@
+
+GEM_NAME = sengi
+ALL_TARGETS_EXT = tmp run init
+
+include Makefile.common
+
+dev:
+	RUBYOPT=-rbundler/setup ruby ./bin/crawler -q http://dev.fox21.at/sengi/
+
+run:
+	$(MKDIR) $@
+
+.PHONY: reset
+reset:
+	RUBYOPT=-rbundler/setup ruby ./bin/config --reset
+
+.PHONY: init
+init:
+	RUBYOPT=-rbundler/setup ruby ./bin/config --init
+
+.PHONY: test
+test:
+	RUBYOPT=-w $(BUNDLER) exec ./tests/ts_all.rb

data/Makefile.common

@@ -0,0 +1,58 @@
+
+# Ruby Common Big
+# 2016-04-09
+
+MV = mv -nv
+RM = rm -rf
+MKDIR = mkdir -p
+CHMOD = chmod
+BUNDLER = bundle
+BUNDLER_OPTIONS = --jobs=5 --retry=3
+GEMSPEC_FILE = $(GEM_NAME).gemspec
+
+.PHONY: all
+all: setup $(ALL_TARGETS_EXT)
+
+.PHONY: setup
+setup: .setup
+
+.setup:
+	$(BUNDLER) install $(BUNDLER_OPTIONS)
+	touch $@
+
+.PHONY: install
+install:
+	gem_file=$$(gem build $(GEMSPEC_FILE) | grep 'File:' | tail -1 | awk '{ print $$2 }'); \
+	sudo gem install $$gem_file; \
+	$(RM) $$gem_file
+
+.PHONY: uninstall
+uninstall:
+	sudo gem uninstall $(GEM_NAME)
+
+.PHONY: update
+update:
+	$(BUNDLER) update
+
+.PHONY: clean
+clean:
+	$(RM) .bundle
+	$(RM) .setup
+	$(RM) Gemfile.lock
+
+.PHONY: release
+release: | releases
+	set -e; \
+	gem_file=$$(gem build $(GEMSPEC_FILE) | grep 'File:' | tail -1 | awk '{ print $$2 }'); \
+	dst="releases/$$gem_file"; \
+	[ ! -f $$dst ]; \
+	$(MV) $$gem_file releases; \
+	gem push $$dst; \
+	echo 'done'
+
+releases:
+	$(MKDIR) $@
+
+tmp:
+	$(MKDIR) $@
+	$(CHMOD) u=rwx,go-rwx $@

data/README.md

@@ -0,0 +1,59 @@
+# Sengi Web Crawler
+
+A web crawler using Ruby and Redis.
+
+## Install
+
+First, run:
+
+	gem install rake bundler nokogiri hiredis
+	make
+
+## Setup
+
+[Redis](http://redis.io/) is used to store everything. So it's always be needed to run Sengi.
+
+Start Redis:
+
+	./bin/redis
+
+Start [Resque](https://github.com/resque/resque) -- Scheduler and Worker:
+
+	./bin/resque_scheduler_start
+	./bin/resque_crawler_start
+
+To get a Resque web dashboard at <http://localhost:8282>, run:
+
+	./bin/resque_server
+
+Init Sengi. This sets default variables to Redis and a blacklist of the deepweb.
+
+	RUBYOPT=-rbundler/setup ruby ./bin/config --init
+
+## Usage
+
+### Queue
+
+To queue a URL to be crawled, run:
+
+	RUBYOPT=-rbundler/setup ruby ./bin/crawler -q http://example.com
+
+### Relative Links Only
+
+To crawl only relative links on `example.com`:
+
+	RUBYOPT=-rbundler/setup ruby ./bin/crawler -r http://example.com
+
+### Serial
+
+Crawl only one URL at a time. The latest datetime will be stored into Redis key `urls:schedule:last`. A new URL to crawl will be scheduled for a new datetime calculated by `urls:schedule:last + url_delay`. Where `url_delay` is the number of seconds between the scheduled URLs.
+
+	RUBYOPT=-rbundler/setup ruby ./bin/crawler -s http://example.com
+
+## License
+
+Copyright (C) 2016 Christian Mayer <http://fox21.at>
+
+This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see <http://www.gnu.org/licenses/>.

data/Rakefile

@@ -0,0 +1,25 @@
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'resque'
+require 'resque/tasks'
+require 'resque-scheduler'
+require 'resque/scheduler/tasks'
+
+require 'sengi'
+
+namespace :resque do
+	task :setup do
+		puts 'resque setup'
+		#require 'resque'
+		Resque.redis = '127.0.0.1:7000'
+	end
+	
+	task :setup_schedule => :setup do
+		puts 'schedule setup'
+		#require 'resque-scheduler'
+	end
+	
+	task :scheduler => :setup_schedule
+end

data/bin/config

@@ -0,0 +1,148 @@
+#!/usr/bin/env ruby
+# coding: UTF-8
+
+require 'optparse'
+require 'hiredis'
+require 'sengi'
+
+
+@redis = Hiredis::Connection.new
+@redis.connect('127.0.0.1', 7000)
+@redis.write(['SELECT', 1])
+@redis.read
+
+def reset
+	(0..1).each do |n|
+		@redis.write(['SELECT', n])
+		@redis.read
+		@redis.write(['FLUSHDB'])
+		puts "FLUSH DB #{n}:         #{@redis.read}"
+	end
+end
+
+def init
+	@redis.write(['SET', 'urls:schedule:lock', 0])
+	puts "urls schedule lock: #{@redis.read}"
+	
+	@redis.write(['SET', 'urls:schedule:last', (Time.now + 180).strftime('%F %T %z')])
+	puts "urls schedule last: #{@redis.read}"
+	
+	@redis.write(['SET', 'urls:delay', TheFox::Sengi::URL_DELAY])
+	puts "urls delay:         #{@redis.read}"
+	
+	@redis.write(['SET', 'urls:separatedelay', TheFox::Sengi::URL_SEPARATE_DELAY])
+	puts "urls separatedelay: #{@redis.read}"
+	
+	@redis.write(['SET', 'urls:reschedule', TheFox::Sengi::URL_RESCHEDULE])
+	puts "urls reschedule:    #{@redis.read}"
+	
+	@redis.write(['SADD', 'domains:ignore',
+		'4chan.org',
+		'about.me',
+		'amazon',
+		'ask.fm',
+		'bitbucket.org',
+		'bit.ly', 'bitly.com',
+		'bbc.com',
+		'blockchain.info',
+		'blogger.com',
+		'blogspot',
+		'cnet.com',
+		'cnn.com',
+		'delicious.com',
+		'digg.com',
+		'disqus.com',
+		'doodle.com',
+		'dropbox.com',
+		'droplr.com',
+		'duckduckgo.com',
+		'ebay.com',
+		'facebook.com', 'fb.com', 'fb.me',
+		'flickr.com',
+		'getpocket.com',
+		'github.com',
+		'google',
+		'gravatar.com',
+		'imdb.com',
+		'imgur.com',
+		'instagram.com',
+		'jsbin.com',
+		'jsfiddle.net',
+		'keybase.io',
+		'kickstarter.com',
+		'linkedin.com',
+		'localhost',
+		'myspace.com',
+		'npmjs.com',
+		'openstreetmap.org', 'osm.org',
+		'packagist.org',
+		'pastebin.com',
+		'paypal.com',
+		'reddit.com',
+		'skype.com',
+		'slack.com',
+		'slashdot.org',
+		'soundcloud.com',
+		'thepiratebay',
+		'tumblr.com',
+		'twitpic.com',
+		'twitter.com',
+		'vimeo.com',
+		'wikipedia.org',
+		'willhaben.at',
+		'ycombinator.com',
+		'xing.com',
+		'yahoo.com',
+		'youtube',
+		])
+	puts "domains ignore:     #{@redis.read}"
+end
+
+@options = {
+}
+opts = OptionParser.new do |o|
+	o.banner = 'Usage: config [options] <command>'
+	o.separator('')
+	
+	o.on_tail('--reset', 'Reset all.') do
+		reset
+		exit
+	end
+	
+	o.on_tail('--init', 'Set up all initial keys.') do
+		init
+		exit
+	end
+	
+	o.on_tail('--reinit', 'Same as --reset and --init.') do
+		reset
+		init
+		exit
+	end
+	
+	o.on_tail('-h', '--help', 'Show this message.') do
+		puts o
+		puts
+		puts 'Commands'
+		puts '  domain'
+		exit 3
+	end
+end
+ARGV << '-h' if ARGV.count == 0
+commands = opts.parse(ARGV)
+command = commands.shift
+
+if command == 'domain'
+	subcommand = commands.shift
+	
+	if subcommand == 'ignore'
+		subsubcommand = commands.shift
+		
+		if subsubcommand == 'add'
+			commands.each do |domain|
+				@redis.write(['SADD', 'domains:ignore', domain.downcase])
+				puts @redis.read
+			end
+		end
+	end
+end

data/bin/crawler

@@ -0,0 +1,64 @@
+#!/usr/bin/env ruby
+# coding: UTF-8
+
+require 'optparse'
+require 'resque'
+require 'resque-scheduler'
+require 'time'
+require 'sengi'
+
+
+@options = {
+	'queue' => false,
+	'serial' => false,
+	'relative' => false,
+	'debug' => false,
+}
+opts = OptionParser.new do |o|
+	o.banner = 'Usage: crawler [options] <url...>'
+	o.separator('')
+	
+	o.on('-q', '--queue', 'Enqueue a URL.') do
+		@options['queue'] = true
+	end
+	
+	o.on('-s', '--serial', 'Schedule the URLs serial.') do
+		# Set this option to true to schedule the URLs serial.
+		# The Redis key 'urls:schedule:last' will be used to store the last
+		# used schedule time and URL_DELAY will be added to create a new
+		# schedule time for the new URL.
+		# Otherwise if this option isn't used URL_DELAY will be added
+		# to the current time.
+		@options['serial'] = true
+	end
+	
+	o.on('-r', '--relative', 'Follow only relative links.') do
+		# And also URLs with the same host.
+		@options['relative'] = true
+	end
+	
+	o.on('-f', '--force', 'Force a URL to be requested.') do
+		@options['force'] = true
+	end
+	
+	o.on('-d', 'Debug') do
+		@options['debug'] = true
+	end
+	
+	o.on_tail('-h', '--help', 'Show this message.') do
+		puts o
+		puts
+		exit 3
+	end
+end
+ARGV << '-h' if ARGV.count == 0
+urls = opts.parse(ARGV)
+
+Resque.redis = '127.0.0.1:7000'
+urls.each_with_index do |url, index|
+	if @options['queue']
+		Resque.enqueue(TheFox::Sengi::CrawlerWorker, url, @options)
+	else
+		TheFox::Sengi::CrawlerWorker.perform(url, @options)
+	end
+end