spider_framework 0.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 78c21ffecbe0e94d8abd0900f5f150e01406b0f6e3259d21d182b0decfdd7fcc
+  data.tar.gz: 5fa7e8c00333f1980ba7429116c8ed45d7458cc5f761c6f918db646024e2dcad
+SHA512:
+  metadata.gz: 4f36eb2f29f6072915025106b6da1536209d08b135f5a8366299afffd0a9f43e00122c34116a0d6d5e3e93b04243f843e3f845dee2ac4405dc22d6567625e93c
+  data.tar.gz: efd5784653d9e802afae75947646aef98dc8bac385fbaf66a849d1804752c7eb67a0c9f0b80fc5ad5e6f077f58e2f7e3005949fe0209bb829da2f15621912480

data/bin/spider.rb

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'logger'
+require_relative '../lib/spider_framework'
+
+options = {
+  log_level: Logger::DEBUG,
+  log_only: false
+}
+
+ARGV.unshift('-h') if ARGV.empty?
+OptionParser.new do |opts|
+  opts.banner = <<~BANNER
+    执行一个爬虫文件
+    用法：spider [options] file
+
+  BANNER
+
+  opts.on("-l", "--log-level=LEVEL", "设置日志级别，可选值包括 FATAL、ERROR、WARN、INFO、DEBUG（默认）") do |level|
+    if %w[FATAL ERROR WARN INFO DEBUG].include?(level)
+      options[:log_level] = Logger.const_get(level)
+    else
+      raise OptionParser::InvalidArgument, level
+    end
+  end
+
+  opts.on("-n", "--log-only=A,B,C", Array, "只允许设置的日志器名称在控制台显示") do |names|
+    options[:log_only] = names
+  end
+end.parse!
+
+SpiderFramework::EnhancedLogger.level = options[:log_level]
+SpiderFramework::EnhancedLogger.names_only = options[:log_only]
+
+program_name = ARGV.shift
+load program_name

data/lib/enhanced_logger.rb

@@ -0,0 +1,37 @@
+# 将本模块加入到目标类里，可生成类方法和实例方法 logger，它返回绑定到目标类的日志。
+require 'logger'
+
+module SpiderFramework::EnhancedLogger
+
+  @level = ::Logger::DEBUG
+
+  class << self
+    attr_accessor :level, :names_only
+
+    def included(base)
+      # 构造绑定类的 @logger 实例
+      base.class_eval do
+        name = base.name
+        names_only = EnhancedLogger.names_only
+        if !names_only || names_only.include?(name)
+          @logger = Logger.new(STDOUT)
+          @logger.level = EnhancedLogger.level
+          original_formatter = Logger::Formatter.new
+          @logger.formatter = proc { |severity, datetime, progname, msg|
+            "[#{datetime.strftime('%F %T')}] #{severity} -- #{name}: #{msg}\n"
+          }
+        else
+          @logger = Logger.new(IO::NULL)
+        end
+      end
+
+      base.singleton_class.class_eval do
+        attr_accessor :logger
+      end
+    end
+  end
+
+  def logger
+    self.class.logger
+  end
+end

data/lib/scheduler.rb

@@ -0,0 +1,53 @@
+##通用调度器
+#
+# 调度器接受具有 call 或 start 方法的对象，并按照先入先出的顺序执行。
+
+class SpiderFramework::Scheduler
+  def initialize
+    @tasks = []
+    @item_handlers = {}
+  end
+
+  def loop
+    until @tasks.empty?
+      task = @tasks.shift # Ruby queue performance: O(1)
+
+      callable, block = [task[:callable], task[:block]]
+      method = callable.respond_to?(:start) ? :start : :call
+      block ? callable.send(method, &block) : callable.send(method)
+
+      # 不执行任何逻辑了
+    end
+  end
+
+  def push(callable, &block)
+    raise NoMethodError, "undefined method `call' or `start' for #{callable}" \
+      unless callable.respond_to?(:call) || callable.respond_to?(:start)
+
+    @tasks.push(callable: callable, block: block)
+  end
+
+  alias :<< :push
+
+  def add_item(type, item, &block)
+    handler = @item_handlers[type.to_s.to_sym]
+    raise "No handler found for type #{type.inspect}" unless handler
+
+    raise NoMethodError, "undefined method `call' or `handle' for #{handler}" \
+      unless handler.respond_to?(:handle) || handler.respond_to?(:call)
+    method = handler.respond_to?(:handle) ? :handle : :call
+
+    p = proc do
+      block ? handler.send(method, item, &block) : handler.send(method, item)
+    end
+
+    push(p)
+  end
+
+  def add_item_handler(type, handler)
+    type = type.to_s.to_sym
+    raise 'Handler already set' if @item_handlers.key?(type)
+
+    @item_handlers[type] = handler
+  end
+end

data/lib/spider_framework.rb

@@ -0,0 +1,4 @@
+module SpiderFramework
+  require_relative 'scheduler'
+  require_relative 'enhanced_logger'
+end

metadata

@@ -0,0 +1,47 @@
+--- !ruby/object:Gem::Specification
+name: spider_framework
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+platform: ruby
+authors:
+- run27017
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-08-13 00:00:00.000000000 Z
+dependencies: []
+description: A lightweight ruby spider framework.
+email: run27017@126.com
+executables:
+- spider.rb
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/spider.rb
+- lib/enhanced_logger.rb
+- lib/scheduler.rb
+- lib/spider_framework.rb
+homepage: https://gitee.com/run27017/spider_framework
+licenses:
+- LGPL-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Spider Framework
+test_files: []