crawl_station 0.1.1

data/lib/crawl_station/fundation/parse_struct.rb

@@ -0,0 +1,45 @@
+require 'ostruct'
+module CrawlStation
+  class ParseStruct
+    extend Forwardable
+    attr_accessor :parse
+
+    %w(namespace parser item link).each do |method_name|
+      define_method(method_name) { get_value(method_name) }
+      define_method("#{method_name}=") { |v| set_value(method_name, v) }
+    end
+
+    def initialize(opts = {})
+      @parse = opts
+      @parse.deep_symbolize_keys!
+    end
+
+    def parser_class
+      path = "#{namespace}/parser/#{parser}"
+      path.camelize.constantize
+    end
+
+    def item_class
+      path = "#{namespace}/item/#{parser}"
+      path.camelize.constantize
+    end
+
+    def [](item)
+      get_value(item)
+    end
+
+    def []=(item, value)
+      set_value(item, value)
+    end
+
+    private
+
+    def get_value(item)
+      @parse[item.to_sym]
+    end
+
+    def set_value(item, value)
+      @parse[item.to_sym] = value
+    end
+  end
+end

data/lib/crawl_station/launcher.rb

@@ -0,0 +1,28 @@
+module CrawlStation
+  class Launcher
+    include Celluloid
+    include Celluloid::Notifications
+
+    def initialize
+      @exit_sign = false
+      subscribe 'exit_launcher', :exit_message
+    end
+
+    def start
+      CS.logger.debug('station starting ... ')
+
+      CS.concurrent_count.times do
+        supervisor = CS::Producer.pool args: [CS.schedule, CS.cache]
+        supervisor.async.start
+      end
+      loop do
+        sleep(10)
+        break if @exit_sign
+      end
+    end
+
+    def exit_message(_topic, data)
+      @exit_sign = data
+    end
+  end
+end

data/lib/crawl_station/logger.rb

@@ -0,0 +1,12 @@
+module CrawlStation
+  module Logger
+    class << self
+      attr_accessor :logger
+      def method_missing(method_name, *args, &block)
+        return false if logger.nil?
+        return super unless logger.respond_to?(method_name)
+        logger.send(method_name, *args, &block)
+      end
+    end
+  end
+end

data/lib/crawl_station/producer.rb

@@ -0,0 +1,64 @@
+module CrawlStation
+  class Producer
+    include Celluloid
+
+    attr_accessor :schedule, :cache, :proxies, :proxy
+
+    def initialize(schedule, cache, proxies = nil)
+      @schedule = schedule
+      @cache = cache
+      @proxies = proxies
+    end
+
+    def start
+      loop { break unless loop_parser }
+      Logger.debug "#{self} done"
+    end
+
+    def loop_parser
+      return sleep(0.2) || true if @schedule.empty?
+      item = @schedule.pop
+      item = CS::ParseStruct.new(item) if item.is_a?(Hash)
+      return sleep(0.2) || true if parsed?(item)
+      Logger.debug "start parse #{item.link}"
+      data = parse_item(item)
+      return true if data.nil? || data.empty?
+      data = parse_links(data, item.namespace)
+      return true if data.empty?
+      item.item_class.new.save(item.link, data)
+      true
+    end
+
+    def parse_item(item)
+      data = cache(item) { item.parser_class.new.crawl(item.link) }
+      @schedule.done(item)
+      data
+    rescue Exception => e
+      Logger.error("%s: %s\n%s" % [item.link, e.message, e.backtrace[0..10].join("\n")])
+      @schedule.failed(item)
+      nil
+    end
+
+    def parse_links(data, namespace)
+      links = ->(data, namespace) do
+        next if data['link'].blank? || parsed?(data)
+        @schedule.push ParseStruct.new(parser: data['parser'], link: data['link'], namespace: namespace)
+      end
+      ['pages', 'details'].each do |field|
+        data.delete(field)&.map { |page| links.call(page, namespace) }
+      end
+      data
+    end
+
+    def parsed?(data)
+      data.nil? || @cache.include?(data['link'])
+    end
+
+    def cache(item, data = 'parsing')
+      @cache[item['link']] = data
+      data = yield if block_given?
+      @cache[item['link']] = data
+      data
+    end
+  end
+end

data/lib/crawl_station/ruby_version_check.rb

@@ -0,0 +1,9 @@
+if ENV['RUBY_VERSION'] < '2.3.0'
+  abort <<-end_message
+CrawlStation require ruby 2.3.0 or newer.
+your are running
+#{ENV['RUBY_VERSION']}
+
+please upgrade to Ruby 2.3.0 or newer to continue.
+  end_message
+end

data/lib/crawl_station/schedule.rb

@@ -0,0 +1,13 @@
+module CrawlStation
+  module Schedule
+    extend self
+    extend Forwardable
+    include Concerns::AdapterConcern
+
+    def adapter=(adapter_name)
+      custom_adapter(adapter_name, CrawlStation::ScheduleAdapters)
+    end
+
+    def_delegators :@adapter, :push, :pop, :empty?, :include?
+  end
+end

data/lib/crawl_station/schedule_adapters/abstract_adapter.rb

@@ -0,0 +1,18 @@
+module CrawlStation
+  module ScheduleAdapters
+    class AbstractAdapter
+      def push(_item)
+      end
+      def pop
+      end
+      def empty?
+      end
+
+      def failed(item)
+      end
+
+      def done(item)
+      end
+    end
+  end
+end

data/lib/crawl_station/schedule_adapters/db_adapter.rb

@@ -0,0 +1,18 @@
+module CrawlStation
+  module ScheduleAdapters
+    class DbAdapter < AbstractAdapter
+      def push(_item)
+      end
+      def pop
+      end
+      def empty?
+      end
+
+      def failed(_item)
+      end
+
+      def done(_item)
+      end
+    end
+  end
+end

data/lib/crawl_station/schedule_adapters/memory_adapter.rb

@@ -0,0 +1,21 @@
+module CrawlStation
+  module ScheduleAdapters
+    class MemoryAdapter < AbstractAdapter
+      def initialize
+        @queue = Queue.new
+        @queue.extend(MonitorMixin)
+        %w(push pop empty?).each do |method_name|
+          self.define_singleton_method method_name do |*args|
+            @queue.synchronize do
+              @queue.send(method_name, *args)
+            end
+          end
+        end
+      end
+
+      def failed(item)
+        CS.logger.debug "#{item.link} failed"
+      end
+    end
+  end
+end

data/lib/crawl_station/tasks/db.rake

@@ -0,0 +1,33 @@
+namespace :db do
+  def db_operation(operator)
+    Rake::Task["db:#{operator}"].invoke('crawl_station')
+    Dir["#{CrawlStation.root}/module/*"].each do |dir|
+      module_name = dirr.split('/').last
+      Rake::Task["db:#{operator}"].invoke(module_name)
+    end
+  end
+
+  desc 'db:create[module_name] if module_name is nil, create all module databases'
+  task :create, [:module_name] => :environment do |_t, args|
+    m_name = args[:module_name]
+    return db_operation(:create) if m_name.nil?
+    m_name = nil if m_name == 'crawl_station'
+    CrawlStation::Utils.create_database(m_name)
+  end
+
+  desc 'db:migrate[module_name] if module_name is nil, migrate all module migrations'
+  task :migrate, [:module_name] => :environment do |_t, args|
+    version = ENV['VERSION']
+    module_name = args[:module_name]
+    return db_operation(:migrate) if module_name.nil?
+    path = "#{CrawlStation::Utils.module_path(module_name)}/db/migrate"
+    path = "#{CrawlStation::Utils.gems_path}/db/migrate" if module_name == 'crawl_station'
+    ActiveRecord::Migrator.migrate(path, version ? version.to_i : nil)
+  end
+
+  task :environment, [:module_name] do |_t, args|
+    config = CrawlStation::Utils.database_config args[:module_name]
+    ActiveRecord::Base.logger = CrawlStation.logger
+    ActiveRecord::Base.establish_connection config
+  end
+end

data/lib/crawl_station/tasks/launcher.rake

@@ -0,0 +1,6 @@
+desc 'launch station system'
+task :launch do
+  CrawlStation::Launcher.new.start
+end
+
+task default: %w(launch)

data/lib/crawl_station/tasks/module.rake

@@ -0,0 +1,6 @@
+require 'rake'
+
+desc 'show module list'
+task :module_list do
+  Dir['module/*'].each { |dir| puts dir.split('/').last }
+end

data/lib/crawl_station/tasks.rb

@@ -0,0 +1,3 @@
+require 'rake'
+
+Dir['tasks/*.rake'].each { |f| load f }

data/lib/crawl_station/templates/create/Gemfile

@@ -0,0 +1,11 @@
+source 'https://rubygems.org'
+
+gem 'rake'
+gem 'thor'
+gem 'celluloid', github: 'celluloid/celluloid', submodules: true
+gem 'wombat', github: 'watsy0007/wombat'
+gem 'crawl_station'
+
+group :development do
+  gem 'pry'
+end

data/lib/crawl_station/templates/create/Rakefile

@@ -0,0 +1,2 @@
+require_relative './config/boot'
+CrawlStation.load_tasks

data/lib/crawl_station/templates/create/config/boot.rb

@@ -0,0 +1,6 @@
+require 'rubygems'
+require 'bundler'
+Bundler.setup(:default)
+require 'wombat'
+require 'crawl_station'
+CrawlStation.boot

data/lib/crawl_station/templates/create/config/initializers/station_config.rb

@@ -0,0 +1,4 @@
+CrawlStation.config do |config|
+  config.adapter = :memory
+  config.parsers = []
+end

data/lib/crawl_station/templates/generate/item.erb

@@ -0,0 +1,9 @@
+module <%= module_class_name %>
+  module Item
+    class <%= class_name %>
+      def save(link, data)
+        puts "#{link}\n#{data}"
+      end
+    end
+  end
+end

data/lib/crawl_station/templates/generate/migration.erb

@@ -0,0 +1,4 @@
+class <%= class_name %> < ActiveRecord::Migration[5.0]
+  def change
+  end
+end

data/lib/crawl_station/templates/generate/parser.erb

@@ -0,0 +1,10 @@
+module <%= module_class_name %>
+  module Parser
+    class <%= class_name %>
+      include Wombat::Crawler
+
+      # title xpath: "//div[@class='title']"
+      # sub_title css: 'div.sub_title'
+    end
+  end
+end

data/lib/crawl_station/utils.rb

@@ -0,0 +1,36 @@
+require 'yaml'
+require 'active_support/core_ext'
+module CrawlStation
+  module Utils
+    class << self
+      def module_path(module_name)
+        "#{CS.root}/module/#{module_name}"
+      end
+
+      def database_path(module_name = nil)
+        global_db_path = "#{CS.root}/config/database.yml"
+        return global_db_path if module_name.nil?
+        path = "#{module_path(module_name)}/config/database.yml"
+        return path if File.exist?(path)
+        global_db_path
+      end
+
+      def database_config(module_name = nil)
+        result = ERB.new(IO.read(database_path(module_name))).result
+        ::YAML.load(result).deep_symbolize_keys[CS.env.to_sym]
+      end
+
+      def templates_path
+        "#{gem_path}/crawl_station/templates"
+      end
+
+      def template_filepath(path)
+        "#{templates_path}/#{path}"
+      end
+
+      def gem_path
+        File.expand_path('../../', __FILE__)
+      end
+    end
+  end
+end

data/lib/crawl_station/version.rb

@@ -0,0 +1,3 @@
+module CrawlStation
+  VERSION = '0.1.1'
+end

data/lib/crawl_station.rb

@@ -0,0 +1,152 @@
+require 'crawl_station/version'
+require 'pathname'
+require 'active_support'
+require 'active_record'
+require 'active_support/dependencies/autoload'
+require 'logger'
+require 'thor'
+require 'celluloid/debug'
+require 'celluloid/current'
+module CrawlStation # :nodoc:
+  extend ActiveSupport::Autoload
+
+  autoload :Configuration
+  autoload :Logger
+  autoload :Utils
+  autoload :ApplicationRecord
+  autoload :Producer
+  autoload :Launcher
+  autoload :Cache
+  autoload :Schedule
+  autoload :ParseStruct, 'crawl_station/fundation/parse_struct'
+  autoload :Command
+
+  module ScheduleAdapters
+    extend ActiveSupport::Autoload
+
+    autoload :AbstractAdapter
+    autoload :MemoryAdapter
+    autoload :DbAdapter
+  end
+
+  module Command
+    extend ActiveSupport::Autoload
+
+    autoload :Create
+    autoload :Generate
+  end
+
+  module CacheAdapters
+    extend ActiveSupport::Autoload
+
+    autoload :AbstractAdapter
+    autoload :MemoryAdapter
+    autoload :DbAdapter
+  end
+
+  module Concerns
+    extend ActiveSupport::Autoload
+
+    autoload :AdapterConcern
+  end
+
+  class << self
+    def config
+      yield @config if block_given?
+    end
+
+    def env
+      @_env ||= ActiveSupport::StringInquirer.new(ENV['CRAWL_STATION_ENV'] || 'development')
+    end
+
+    def env=(environment)
+      @_env = ActiveSupport::StringInquirer.new(environment)
+    end
+
+    def root
+      Pathname.new(File.expand_path('.'))
+    end
+
+    def logger
+      @_logger ||= begin
+                     CrawlStation::Logger.logger ||= ::Logger.new(STDERR) do |log|
+                       log.level = ::Logger.DEBUG
+                     end
+                   end
+    end
+
+    def logger=(logger)
+      @_logger = CrawlStation::Logger.logger = logger
+    end
+
+    def concurrent_count
+      @_concurrent_count ||= ENV['CRAWL_STATION_CONCURRENT_COUNT'] || 1
+    end
+
+    def concurrent_count=(count)
+      @_concurrent_count = count
+    end
+
+    def schedule
+      Schedule.adapter
+    end
+
+    def schedule=(item)
+      Schedule.adpater(item)
+    end
+
+    def cache
+      Cache.adapter
+    end
+
+    def cache=(item)
+      Cache.adapter(item)
+    end
+
+    def proxies
+    end
+
+    def proxies(item)
+    end
+
+    def load_tasks
+      task_path = File.expand_path('../', __FILE__)
+      [
+        "#{task_path}/crawl_station/tasks/*.rake",
+        "#{CrawlStation.root}/lib/tasks/**/*.rake"
+      ].each { |path| Dir[path].each { |f| load f } }
+    end
+
+    def load_modules
+      %w(item parser config).each do |path|
+        Dir["#{CS.root}/module/*/#{path}/**/*.rb"].each { |f| require f }
+      end
+    end
+
+    def init_application
+      @config ||= CrawlStation::Configuration
+      Dir["#{CS.root}/config/initializers/**/*.rb"].each { |f| require f }
+    end
+
+    def config_adapter
+      adapter = @config.adapter || 'memory'
+      schedule = adapter
+      cache = adapter
+    end
+
+    def config_parsers
+      parsers = @config.parsers || []
+      parsers.each { |p| schedule.push p }
+    end
+
+    def boot
+      init_application
+      load_modules
+      config_adapter
+      config_parsers
+    end
+  end
+end
+
+CS = CrawlStation
+Celluloid.boot