news_crawler 0.0.0.pre.1

data/lib/news_crawler/url_helper.rb

@@ -0,0 +1,50 @@
+#--
+# NewsCrawler - a website crawler
+#
+# Copyright (C) 2013 - Hà Quang Dương <contact@haqduong.net>
+#
+# This file is part of NewsCrawler.
+#
+# NewsCrawler is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# NewsCrawler is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with NewsCrawler.  If not, see <http://www.gnu.org/licenses/>.
+#++
+
+module NewsCrawler
+  # Contains various method for processing url
+  module URLHelper
+    # produce true if 2 urls belong to same domain
+    # @param  [ String  ] url1 Url 1
+    # @param  [ String  ] url2 Url 2
+    # @return [ Boolean ] true if both url belong to same domain
+    def same_domain?(url1, url2)
+      p1 = get_url_path(url1)
+      p2 = get_url_path(url2)
+      d1 = p1[:domain].split('.').reverse
+      d2 = p2[:domain].split('.').reverse
+      d1.zip(d2).inject(true) do | mem, obj |
+        mem = mem && ((obj[0] == obj[1]) || (obj[0].nil? || obj[1].nil?))
+      end
+    end
+
+    # split URL into 3 parts: scheme, domain, path
+    # @param [ String ] url
+    # return [ Hash   ] contains parts
+    def get_url_path(url)
+      pattern = /((?<scheme>(http|https)):\/\/)?(?<domain>[^\/]+)?(?<path>\/.*)?/
+      md = pattern.match(url)
+      { :scheme => md[:scheme],
+        :domain => md[:domain],
+        :path => md[:path]}
+    end
+  end
+end

data/lib/news_crawler/utils/robots_patch.rb

@@ -0,0 +1,34 @@
+#--
+# NewsCrawler - a website crawler
+#
+# Copyright (C) 2013 - Hà Quang Dương <contact@haqduong.net>
+#
+# This file is part of NewsCrawler.
+#
+# NewsCrawler is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# NewsCrawler is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with NewsCrawler.  If not, see <http://www.gnu.org/licenses/>.
+#++
+
+require 'robots'
+require 'uri'
+require 'singleton'
+
+class Robots
+  include Singleton
+  attr_accessor :user_agent
+
+  def initialize
+    @user_agent = 'NewsCrawler'
+    @parsed = {}
+  end
+end

data/lib/news_crawler.rb

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+#--
+# NewsCrawler - a website crawler
+#
+# Copyright (C) 2013 - Hà Quang Dương <contact@haqduong.net>
+#
+# This file is part of NewsCrawler.
+#
+# NewsCrawler is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# NewsCrawler is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with NewsCrawler.  If not, see <http://www.gnu.org/licenses/>.
+#++
+
+# TODO implement easy API
+
+require 'news_crawler/autostart'
+require 'news_crawler/config'
+require 'news_crawler/downloader'
+require 'news_crawler/link_selector/same_domain_selector'
+
+NewsCrawler::Storage::RawData.set_engine(:mongo)
+NewsCrawler::Storage::URLQueue.set_engine(:mongo)
+
+include NewsCrawler::Storage
+
+URLQueue.clear
+
+# RawData.clear
+# dwl = NewsCrawler::Downloader.new
+# dwl.run
+# #dwl.async.run
+# #dwl.graceful_terminate
+
+URLQueue.mark_all('NewsCrawler::LinkSelector::SameDomainSelector', "unprocessed")
+
+puts "Raw entries: #{RawData.count}"
+NewsCrawler::LinkSelector::SameDomainSelector.new
+puts "URL entries: #{URLQueue.count}"

metadata

@@ -0,0 +1,203 @@
+--- !ruby/object:Gem::Specification
+name: news_crawler
+version: !ruby/object:Gem::Version
+  version: 0.0.0.pre.1
+platform: ruby
+authors:
+- Hà Quang Dương
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-07-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mongo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: celluloid
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.14'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.14'
+- !ruby/object:Gem::Dependency
+  name: simpleconfig
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: robots
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.14'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A flexible, modular web crawler
+email: contact@haqduong.net
+executables:
+- news_crawler
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/news_crawler.rb
+- lib/news_crawler/autostart.rb
+- lib/news_crawler/config.rb
+- lib/news_crawler/crawler_module.rb
+- lib/news_crawler/downloader.rb
+- lib/news_crawler/link_selector/same_domain_selector.rb
+- lib/news_crawler/nc_logger.rb
+- lib/news_crawler/storage/raw_data.rb
+- lib/news_crawler/storage/raw_data/mongo_storage.rb
+- lib/news_crawler/storage/raw_data/raw_data_engine.rb
+- lib/news_crawler/storage/url_queue.rb
+- lib/news_crawler/storage/url_queue/mongo_storage.rb
+- lib/news_crawler/storage/url_queue/url_queue_engine.rb
+- lib/news_crawler/storage/url_queue/url_queue_error.rb
+- lib/news_crawler/url_helper.rb
+- lib/news_crawler/utils/robots_patch.rb
+- lib/news_crawler/default_config.yml
+- lib/news_crawler/default_sds.yml
+- bin/news_crawler
+homepage: http://haqduong.github.io/news_crawler/
+licenses:
+- GPLv3
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: 2.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>'
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: News crawler
+test_files: []