w3map 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in w3map.gemspec
+gemspec
+

data/LICENSE.txt

@@ -0,0 +1 @@
+Copyright (c) 2013 Samuel Sanchez

data/README.md

@@ -0,0 +1,46 @@
+# W3map
+
+w3map is a distributed system for process OSINT.
+Open Source Intelligence (OSINT) is an information processing discipline that involves finding,
+selecting, mining and acquiring information from publicly available sources and analyzing it to produce actionable intelligence/knowledge.
+The term "open" refers to overt, publicly available sources (as opposed to covert or classified sources);
+it is not related to open-source software.
+
+ w3map is in alpha version.
+For the time being:
+  - you can't configure redis server
+  - you can't use your add your processing system
+  - you can't use your use your reporting system
+  - potentially you can discover bugs
+
+## Installation
+
+Install it yourself as:
+
+    $ gem install w3map
+
+## Usage
+
+WARNING: You must have Redis server started in your localhost.
+
+    $ export W3MAP_SESSION=<your-processing-session-name>
+    $ w3map push http://www.example.com
+
+Start bot:
+
+    $ w3map bot # you can start many bots
+
+Session information:
+
+    $ w3map queuelen # show queue length
+    $ w3map queue 5 # show last 5 entries in queue
+    $ w3map urls # show processing urls list and data store
+    $ w3map report ResponseCodeReport # execute standard report
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,7 @@
+require "bundler/gem_tasks"
+
+require 'rspec/core/rake_task'
+
+task :default => :spec
+desc 'Run tests with RSpec'
+RSpec::Core::RakeTask.new(:spec)

data/bin/w3map

@@ -0,0 +1,63 @@
+#!/usr/bin/env ruby
+#encoding: utf-8
+
+$:.unshift(File.dirname(__FILE__) + "/../lib")
+require 'w3map'
+
+@session = W3map::Session.new(ENV['W3MAP_SESSION'])
+
+# TODO: Refactor this !
+def exec_cmd(cmd)
+  case cmd[0]
+  when 'flushall'
+    @session.queue.flush_processing_urls
+    @session.queue.flush_queue
+  when 'queuelen'
+    puts @session.queue.len
+  when 'queue'
+    @session.queue.last_entries(cmd[1].to_i).reverse.each_with_index do |url, idx|
+      puts "[#{idx}] #{url}"
+    end
+  when 'pop'
+    puts @session.queue.pop
+  when 'push'
+    @session.queue.push cmd[1]
+  when 'bot'
+    begin
+      bot = W3map::Bot.new(@session)
+      # TODO: load neurons
+      bot.start 
+    rescue SignalException => e
+      raise e
+    rescue Exception => e
+      puts
+      puts "-- ERROR: #{e.message}"
+      puts "--   #{e.backtrace.first}"
+    end
+  when 'report' # TODO: load reports ...
+    W3map::Reports.const_get(cmd[1]).new(@session).exec
+  when 'urls'
+    @session.queue.processing_urls.each do |url|
+      puts url
+      @session.dstore.read_hash(url).each do |field, value|
+        puts "   #{field}: #{value}"
+      end
+    end
+  else
+    puts "Unknow command: #{cmd[0]}"
+  end
+end
+
+if ARGV[0] == '-i'
+  puts "Welcome on w3map #{W3map::VERSION} #{Time.now}"
+  puts "Session: #{@session.name} loaded"
+  puts
+  loop do
+    print "#{@session.name}>> "
+    cmd = STDIN.gets.strip.split(' ')
+    break if cmd[0] == 'quit' || cmd[0] == 'exit'
+    exec_cmd cmd
+  end 
+else
+  exec_cmd ARGV
+end

data/lib/ext/string.rb

@@ -0,0 +1,11 @@
+class String
+  def to_utf8
+    if 1.8 == RUBY_VERSION.to_f
+      require 'iconv'
+      Iconv.conv('ISO-8859-1//TRANSLIT', 'utf-8', self)
+    else
+      self.encode('UTF-8')
+      #why not self.force_encoding('UTF-8') ???
+    end
+  end
+end

data/lib/w3map.rb

@@ -0,0 +1,179 @@
+require 'w3map/version'
+require 'ext/string'
+
+require 'uri'
+require 'cgi'
+require 'net/http'
+require 'redis'
+require 'nokogiri'
+
+require 'w3map/neurons/neuron'
+require 'w3map/neurons/simple_crawler'
+
+require 'w3map/reports/report'
+require 'w3map/reports/response_code_report'
+
+module W3map
+  REDIS_PREFIX = "w3map"
+  class InvalidSessionName < Exception; end
+
+  class Session
+    attr_reader :name, :queue, :dstore
+    def initialize(name=nil, redis_config={})
+      raise InvalidSessionName if name.nil? || name.length == 0
+      @name = name
+      @redis_config = redis_config
+    end
+    def queue
+      @queue ||= DistributedQueue.new(@name, @redis_config)
+    end
+    def dstore
+      @dstore ||= DataStore.new(@name, @redis_config)
+    end
+  end
+
+  class DataStore
+    def initialize(session_name, config={})
+      @session_name = session_name
+      @redis = Redis.new(config)
+    end
+    def save(key, field, value)
+      @redis.hset schema(key), field, value
+    end
+    def save_hash(key, hash)
+      @redis.hmset schema(key), hash.flatten
+    end
+    def read(key, field)
+      @redis.hget schema(key), field
+    end
+    def read_hash(key)
+      @redis.hgetall schema(key)
+    end
+    def flush
+      @redis.keys(schema('*')).each { |key| @redis.del key }
+    end
+    def close
+      @redis.quit
+    end
+    private
+    def schema(key=nil)
+      key.nil? ? "#{REDIS_PREFIX}:#{@session_name}:dstore" : "#{REDIS_PREFIX}:#{@session_name}:dstore:#{key}"
+    end
+  end
+
+  class DistributedQueue
+    def initialize(session_name, config={})
+      @session_name = session_name
+      @redis = Redis.new config
+    end
+    def push(url)
+      add_to_processing_urls url
+      @redis.lpush queue_schema, url
+    end
+    def add_to_processing_urls(url)
+      @redis.sadd urls_schema, url
+    end
+    def close
+      @redis.quit
+    end
+    def last_entries(limit)
+      limit = limit + -1 if limit > 0
+      @redis.lrange queue_schema, 0, limit
+    end
+    def len
+      @redis.llen queue_schema
+    end
+    def already_processed?(url)
+      @redis.sismember urls_schema, url
+    end
+    def processing_urls
+      @redis.smembers(urls_schema)
+    end
+    def pop
+      _, url = @redis.brpop queue_schema
+      url
+    end
+    def flush_processing_urls
+      @redis.del urls_schema
+    end
+    def flush_queue
+      @redis.del queue_schema
+    end
+    private
+    def queue_schema
+      "#{REDIS_PREFIX}:#{@session_name}:queue"
+    end
+    def urls_schema
+      "#{REDIS_PREFIX}:#{@session_name}:urls"
+    end
+  end
+
+  class Bot
+    attr_reader :session
+    def initialize(session)
+      @session = session
+      @http_engine = HttpEngine.new(self)
+      # TODO: create class NeuronStack (NeuronPipeline?)
+      @neurons = []
+      @neurons << Neurons::SimpleCrawler.new(self)
+    end
+    def start
+      loop do
+        puts "Bot waiting for task (#{Time.now})"
+        url = @session.queue.pop
+        # break if url.nil?
+        puts "PROCESS #{url}"
+        response = @http_engine.response_for(url)
+        # save url, 'response-code', response.code
+        data = {'response-code' => response.code}
+        @neurons.each { |neuron| neuron.process url, response, data } #TODO: use NeuronStack or other
+        @session.dstore.save_hash url, data
+      end
+    rescue SignalException => e
+      puts
+    end
+    def stop
+      # @session.close ?
+    end
+  end
+
+  class HttpEngine
+    attr_reader :bot
+    def initialize(bot)
+      @bot = bot
+    end
+    def response_for(url)
+      uri = URI.parse(url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      if uri.port == 443
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      end
+      get = Net::HTTP::Get.new(uri.request_uri)
+      http.request(get)
+    end
+  end
+
+  module HtmlHelpers
+    module_function
+    def normalize(url)
+      url = url.to_utf8
+      begin
+        uri = URI.parse URI.unescape(url)
+      rescue URI::InvalidURIError
+        uri = URI.parse URI.escape(url)
+      end
+      uri.fragment = nil
+      uri.to_s
+    end
+    def make_absolute(url, href)
+      url, href = url.to_utf8, href.to_utf8
+      a = URI.parse(URI.encode(url)) + URI.encode(href)
+      a.to_s
+    end
+    def is_internal?(url1, url2)
+      url1, url2 = url1.to_utf8, url2.to_utf8
+      URI.parse(URI.encode(url1)).host.to_s.downcase == URI.parse(URI.encode(url2)).host.to_s.downcase
+    end
+  end
+end

data/lib/w3map/neurons/neuron.rb

@@ -0,0 +1,14 @@
+module W3map
+  module Neurons
+
+    class Neuron
+      def initialize(bot)
+        @bot = bot
+      end
+      def process(url, response, data)
+        raise NotImplementedError
+      end
+    end
+
+  end
+end

data/lib/w3map/neurons/simple_crawler.rb

@@ -0,0 +1,24 @@
+module W3map
+  module Neurons
+
+    class SimpleCrawler < Neuron
+      def process(url, response, data)
+        if response.header['Content-Type'] =~ /(.*)text\/html(.*)/
+          doc = Nokogiri::HTML(response.body)
+          data['page-title'] = doc.search('head/title').text || ''
+          doc.css('a').each do |a|
+            unless a[:href].nil?
+              founded_url = HtmlHelpers.normalize HtmlHelpers.make_absolute(url, a[:href])
+              if HtmlHelpers.is_internal?(url, founded_url) && !@bot.session.queue.already_processed?(founded_url)
+                @bot.session.queue.push founded_url 
+              else
+                @bot.session.queue.add_to_processing_urls(founded_url)
+              end
+            end
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/w3map/reports/report.rb

@@ -0,0 +1,14 @@
+module W3map
+  module Reports
+
+    class Report
+      def initialize(session)
+        @session = session
+      end
+      def exec
+        raise NotImplementedError
+      end
+    end
+
+  end
+end

data/lib/w3map/reports/response_code_report.rb

@@ -0,0 +1,25 @@
+module W3map
+  module Reports
+
+    class ResponseCodeReport < Report
+      def exec
+        @session.queue.processing_urls.each do |url|
+          hash = @session.dstore.read_hash(url)
+          puts "[#{hash['response-code']}] #{url}"
+        end
+      end
+    end
+
+    class PageTitleReport < Report
+      def exec
+        @session.queue.processing_urls.each do |url|
+          hash = @session.dstore.read_hash(url)
+          puts 
+          puts url
+          puts "  ## #{hash['page-title']}"
+        end
+      end
+    end
+
+  end
+end

data/lib/w3map/version.rb

@@ -0,0 +1,3 @@
+module W3map
+  VERSION = "0.0.1"
+end

data/spec/html_helpers_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+describe HtmlHelpers do
+
+  it '.normalize' do
+    normalized_url = 'http://www.w3map.org/mypage'
+    HtmlHelpers.normalize('http://www.w3map.org/mypage').should == normalized_url
+    HtmlHelpers.normalize('http://www.w3map.org/mypage%23').should == normalized_url
+    HtmlHelpers.normalize('http://www.w3map.org/mypage#23').should == normalized_url
+    HtmlHelpers.normalize(' http://www.w3map.org/mypage').should == normalized_url
+    HtmlHelpers.normalize('%20http://www.w3map.org/mypage').should == normalized_url
+    HtmlHelpers.normalize('http://www.w3map.org/images/Sublime Text 2.icns').should == 'http://www.w3map.org/images/Sublime%20Text%202.icns'
+    HtmlHelpers.normalize('http://www.foo.com/reseaux/10/05112009/media-sociaux-entreprise-reseaux-professionnels-usage-podcast-facebook-twitter-38937-.html%23xtor=EPR-233-[HTML]-20091105')
+      .should == 'http://www.foo.com/reseaux/10/05112009/media-sociaux-entreprise-reseaux-professionnels-usage-podcast-facebook-twitter-38937-.html'
+  end
+
+  it '.is_internal?' do
+    initial_url = 'http://www.w3map.org'
+    HtmlHelpers.is_internal?(initial_url, 'http://www.w3map.org/foo/bar').should be_true
+    HtmlHelpers.is_internal?(initial_url, 'http://www.w3map.org/foo?bar=baz').should be_true
+    HtmlHelpers.is_internal?(initial_url, 'http://www.google.com').should be_false
+  end
+
+  it '.make_absolute' do
+    initial_url = 'http://www.w3map.org'
+    HtmlHelpers.make_absolute(initial_url, '/foo/bar').should == "#{initial_url}/foo/bar"
+    HtmlHelpers.make_absolute("#{initial_url}/", '/foo/bar').should == "#{initial_url}/foo/bar"
+    HtmlHelpers.make_absolute(initial_url, 'http://www.google.com/foo/bar').should == 'http://www.google.com/foo/bar'
+    HtmlHelpers.make_absolute("#{initial_url}/", 'http://www.google.com/foo/bar').should == 'http://www.google.com/foo/bar'
+  end
+end

data/spec/http_engine_helpers_spec.rb

@@ -0,0 +1,21 @@
+require 'spec_helper'
+
+describe HttpEngine do
+  before(:each) do
+    @bot = stub(Bot)
+    @http_engine = HttpEngine.new(@bot)
+  end
+  it 'have a bot' do
+    @http_engine.bot.should_not be_nil
+    @http_engine.bot.should be(@bot)
+  end
+  it '.response_for' do
+    resp = @http_engine.response_for 'http://www.w3map.org/'
+    resp.should_not be_nil
+    resp.code.should == 200.to_s
+    resp.body.should match /w3map/
+  end
+
+  # TODO:
+  # - test ssl
+end

data/spec/spec_helper.rb

@@ -0,0 +1,5 @@
+require 'rspec'
+
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+require 'w3map'
+include W3map

data/w3map.gemspec

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'w3map/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "w3map"
+  gem.version       = W3map::VERSION
+  gem.authors       = ["Sam"]
+  gem.email         = ["samuel@pagedegeek.com"]
+  gem.description   = %q{w3map is a distributed system for process OSINT.
+    Open Source Intelligence (OSINT) is an information processing discipline that involves finding,
+    selecting, mining and acquiring information from publicly available sources and analyzing it to produce actionable intelligence/knowledge.
+    The term "open" refers to overt, publicly available sources (as opposed to covert or classified sources);
+    it is not related to open-source software.}
+  gem.summary       = %q{Web scanner for gathering, mining, process, improve information open/public sources (OSINT).}
+  gem.homepage      = "http://www.w3map.org"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib", "bin"]
+
+  gem.add_dependency 'redis'
+  gem.add_dependency 'nokogiri'
+
+  gem.add_development_dependency 'rspec'
+end

metadata

@@ -0,0 +1,121 @@
+--- !ruby/object:Gem::Specification
+name: w3map
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Sam
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-02-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ! "w3map is a distributed system for process OSINT.\n    Open Source
+  Intelligence (OSINT) is an information processing discipline that involves finding,\n
+  \   selecting, mining and acquiring information from publicly available sources
+  and analyzing it to produce actionable intelligence/knowledge.\n    The term \"open\"
+  refers to overt, publicly available sources (as opposed to covert or classified
+  sources);\n    it is not related to open-source software."
+email:
+- samuel@pagedegeek.com
+executables:
+- w3map
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/w3map
+- lib/ext/string.rb
+- lib/w3map.rb
+- lib/w3map/neurons/neuron.rb
+- lib/w3map/neurons/simple_crawler.rb
+- lib/w3map/reports/report.rb
+- lib/w3map/reports/response_code_report.rb
+- lib/w3map/version.rb
+- spec/html_helpers_spec.rb
+- spec/http_engine_helpers_spec.rb
+- spec/spec_helper.rb
+- w3map.gemspec
+homepage: http://www.w3map.org
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+- bin
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Web scanner for gathering, mining, process, improve information open/public
+  sources (OSINT).
+test_files:
+- spec/html_helpers_spec.rb
+- spec/http_engine_helpers_spec.rb
+- spec/spec_helper.rb