flute 0.1.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 96163e0495e9bc2f65b308943c132fbb622ef4a1
-  data.tar.gz: 23092154b381dd0a02fe18d5e81aa0ab4a691d4c
+  metadata.gz: 17b89f3543d87eedc0830aa9e3673d45bb4e391f
+  data.tar.gz: feeaa5ae2b1b59701042f662c670d74b558a1822
 SHA512:
-  metadata.gz: 1ce87363fc3566fed6590d9fcebd547bfd0fa1574d22d300bb6f32430a9dd4a20c8cbf761b7b3292b5fbbffad1c5d0a67976e9c315e0ca6785cd7accdc9c73b7
-  data.tar.gz: 570a96fce99075b8f68aa9ab8fd968b8c997caeab78a396f6192f5de4add5d37e26321279d4f7ccf18bbef40e4255ef6126fdf562a5d59e144ec9e9c6c33e45c
+  metadata.gz: 1f97074135702109a1bbbc2f6a1500e1982d4e09c7ee4c1a13c06fcede28e66d7ea1bef78a935625b163a74f479f9c4c2ed2b24ce4aa2ea06b185724f17b4c5b
+  data.tar.gz: ccaec98172277454fd21e79d41b80b95614277f215eae5cd7594d6eb9bff1ebc9c27bb43f8eea047764f815d48b03766318831ba2f86aabb72769ced9d62f414

data/README.md

@@ -9,23 +9,13 @@
 
 ## Installation
 
-Add this line to your application's Gemfile:
-
-```ruby
-gem 'flute'
-```
-
-And then execute:
-
-    $ bundle
-
-Or install it yourself as:
-
     $ gem install flute
 
+之后可以使用 `flute new project_name` 来创建一个项目。
+
 ## Usage
 
-TODO
+项目包含了一个从爱范儿爬虫例子，可以进行参考，或许会有更多的实际例子发布
 
 
 ## License

data/exe/flute

@@ -0,0 +1,51 @@
+#!/usr/bin/env ruby
+
+require 'thor'
+require 'fileutils'
+
+class FluteCLI < Thor
+  desc "new NAME", "create project"
+  def new(name)
+    dirs = [
+      'tmp',
+      'log',
+      'config',
+      'lib',
+      'app',
+      'app/utils',
+    ]
+    create_dir(name, dirs)
+    FileUtils.touch "#{name}/tmp/.keep"
+    FileUtils.touch "#{name}/log/.keep"
+
+    copy_template 'Gemfile', "#{name}/Gemfile"
+    copy_template 'main.rb', "#{name}/main.rb"
+    copy_template 'Rakefile', "#{name}/Rakefile"
+
+    FileUtils.cp_r template_path('items'), "#{name}/app/items"
+    FileUtils.cp_r template_path('spiders'), "#{name}/app/spiders"
+    FileUtils.cp_r template_path('middlewares'), "#{name}/app/middlewares"
+
+    puts "cd #{name}"
+    puts "bundle install"
+    puts "rake start"
+  end
+
+  private
+  def create_dir(project_name, dirs)
+    Dir.mkdir(project_name)
+    dirs.each {|dir|
+      Dir.mkdir "#{project_name}/#{dir}"      
+    }
+  end
+
+  def copy_template(name, dest)
+    FileUtils.cp template_path(name), dest
+  end
+
+  def template_path(name)
+    File.expand_path("../lib/template/#{name}", File.dirname(__FILE__))
+  end
+end
+
+FluteCLI.start(ARGV)

data/flute.gemspec

@@ -17,11 +17,18 @@ Gem::Specification.new do |spec|
   spec.files         = `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
   end
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+
+  spec.bindir      = "exe"
+  spec.executables = ["flute"]
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency "bundler", "~> 1.13"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
+
+  spec.add_dependency 'activesupport', '~> 5.0.1'
+  spec.add_dependency 'typhoeus', '~> 1.1.2'
+  spec.add_dependency 'nokogiri', '~> 1.7.0'
+  spec.add_dependency 'thor'
+
 end

data/lib/flute/document.rb

@@ -0,0 +1,28 @@
+module Flute
+  class Document < Nokogiri::HTML::Document
+    def og_title
+      og('title')
+    end
+
+    def og_description
+      og('description')
+    end
+
+    def og_image
+      og('image')
+    end
+
+    def og_site_name
+      og('site_name')
+    end
+
+    def description
+      self.css('meta[name="description"]')&.first&.attr('content')&.strip
+    end
+
+    def og(name)
+      self.css("meta[property='og:#{name}']")&.first&.attr('content')
+    end
+
+  end
+end

data/lib/flute/network.rb

@@ -0,0 +1,29 @@
+module Flute
+  class Network
+    def get(url, params: {}, headers: {})
+      options = make_options params, headers
+      response = Typhoeus.get(url, options)
+      Response.new response
+    end
+
+    def request(url,  params: {}, headers: {}, method: :get, body: {})
+      options = make_options params, headers
+      request = Request.new url, options.merge(method: method, body: body)      
+      request
+    end
+
+    def make_options(params={}, headers={})
+      options = {
+          headers: headers,
+          params: params,
+          followlocation: true,
+          accept_encoding: 'gzip'
+      }      
+      options
+    end
+
+    def create_manager(max_concurrency: 5)
+      Typhoeus::Hydra.new(max_concurrency: max_concurrency)
+    end
+  end
+end

data/lib/flute/request.rb

@@ -0,0 +1,19 @@
+module Flute
+  class Request < Typhoeus::Request
+    def meta
+      @meta
+    end
+
+    def meta=(item)
+      @meta = item
+    end
+
+    def response_html
+      Document.parse response.body
+    end
+
+    def response_json
+      JSON.parse(response.body).symbolize_keys
+    end
+  end
+end

data/lib/flute/response.rb

@@ -0,0 +1,22 @@
+module Flute
+  class Response
+    attr_reader :status, :body
+    def initialize(response)
+      @status = response.code
+      @body = response.body
+    end
+
+    def success?
+      200 <= @status && @status < 300
+    end
+
+    def parse_html
+      @parse_html ||= Document.parse(@body)
+    end
+
+    def parse_json
+      @parse_json ||= JSON.parse(@body).symbolize_keys
+    end
+
+  end
+end

data/lib/flute/spider_base.rb

@@ -0,0 +1,83 @@
+module Flute
+  class SpiderBase
+
+    def manager_network
+      network
+    end
+
+    def network
+      self.class.opts[:network]
+    end
+
+    def manager
+      @manager
+    end
+
+    def reset_manager
+      @manager = manager_network.create_manager
+    end
+
+    def items
+      []
+    end
+
+    def before_parse(_items)
+      _items
+    end
+
+    def init_queue(_items)
+      reset_manager
+      _items
+    end
+
+    def parse(_items)
+      _items
+    end
+
+    def run_queue(_items)
+      manager.run if manager.queued_requests.size > 0
+      _items
+    end
+
+    def after_parse(_requests)
+      _requests
+    end
+
+    def pipeline
+      methods = self.class.opts[:middleware].map {|name| method(name)}
+      _items = items
+      methods.each { |middleware|
+        _items = middleware.call(_items)
+      }
+      _items
+    end
+
+
+    class << self
+
+      def opts
+        @opts ||= {
+          middleware: [:before_parse, :init_queue, :parse, :run_queue, :after_parse],
+          network: Network.new
+        }
+      end
+
+      def set_name(value)
+        opts[:name] = value
+      end
+
+      def start_urls(value)
+        opts[:start_urls] = value
+      end
+
+      def middleware(value)
+        opts[:middleware] = value
+      end
+
+      def set_network(value)
+        opts[:network] = value
+      end
+
+    end
+  end
+end

data/lib/flute/version.rb

@@ -1,3 +1,3 @@
 module Flute
-  VERSION = "0.1.0"
+  VERSION = "0.2.1"
 end

data/lib/flute.rb

@@ -1,5 +1,15 @@
 require "flute/version"
 
+require 'active_support/all'
+require 'nokogiri'
+require 'typhoeus'
+
+require 'flute/document'
+require 'flute/request'
+require 'flute/response'
+require 'flute/network'
+require 'flute/spider_base'
+
 module Flute
-  # Your code goes here...
+  
 end

data/lib/template/Gemfile

@@ -0,0 +1,14 @@
+source 'https://rubygems.org'
+
+gem 'flute'
+
+gem 'nokogiri'
+gem 'activesupport', require: 'active_support/all'
+gem 'typhoeus'
+
+gem 'virtus'
+gem 'feedjira'
+
+group :development do
+  gem 'pry'
+end

data/lib/template/Rakefile

@@ -0,0 +1,21 @@
+require_relative('./main')
+
+module IRBExtensition
+  def reload!
+    puts 'reloading...'
+    ActiveSupport::Dependencies.clear
+    nil
+  end
+end
+
+task :console do
+  require 'irb'
+  ARGV.clear
+  
+  IRB::ExtendCommandBundle.send :include, IRBExtensition
+  IRB.start
+end
+
+task :start do
+  start_ant
+end

data/lib/template/items/base_item.rb

@@ -0,0 +1,5 @@
+class BaseItem
+  include Virtus.model
+
+  attribute :url, String
+end

data/lib/template/items/rss_item.rb

@@ -0,0 +1,17 @@
+class RssItem < BaseItem
+  attribute :title, String
+  attribute :summary, String
+  attribute :created, Integer
+  attribute :content, String
+  attribute :seed_id, Integer
+  attribute :head_name, String
+  attribute :head_icon, String, default: 'link'
+
+  def self.create(rss_item)
+    RssItem.new title: rss_item.title&.squish,
+                url: rss_item.url,
+                summary: rss_item.summary&.squish,
+                content: rss_item.content&.squish,
+                created: rss_item.published.to_i
+  end
+end

data/lib/template/items/web_item.rb

@@ -0,0 +1,7 @@
+class WebItem < BaseItem
+
+  attribute :title, String
+  attribute :description, String
+  attribute :image, String
+  
+end

data/lib/template/main.rb

@@ -0,0 +1,27 @@
+env = ENV['CRAWLER_ENV'] || 'development'
+AREA = ENV['AREA_ENV'] || :cn
+
+require 'rubygems'
+require 'bundler'
+Bundler.require(:default, env)
+
+$project_root = Pathname.new File.dirname(__FILE__)
+
+load_paths = ['utils', 'items', 'middlewares', 'spiders'].map do |dir|
+  $project_root.join('app', dir)
+end
+
+ActiveSupport::Dependencies.autoload_paths += load_paths
+
+def start_ant
+  spiders = Dir[$project_root.join('app', 'spiders', '*.rb')].map do |path|
+    File.basename(path, '.rb').to_s.classify.constantize
+  end
+  spiders.each do |spiders|
+    if env != 'production'
+      puts spiders.new.pipeline.map &:to_h
+    else
+      spiders.new.pipeline
+    end
+  end
+end

data/lib/template/middlewares/rss_middleware.rb

@@ -0,0 +1,16 @@
+module RssMiddleware
+
+  def rss_items      
+    @rss_body ||= network.get(self.class.opts[:start_urls].first).body
+    @rss ||= Feedjira::Feed.parse @rss_body
+    @rss_items ||= @rss.entries.select {|item|
+      !item.url.blank?
+    }.map {|item|
+      RssItem.create item
+    }
+  end
+
+  def items
+    rss_items[0..5]
+  end
+end

data/lib/template/spiders/ifanr_spider.rb

@@ -0,0 +1,35 @@
+class IfanrSpider < Flute::SpiderBase
+  include RssMiddleware
+  
+  set_name 'ifanr'
+  start_urls ['http://www.ifanr.com/feed']
+
+  def before_parse(items)
+    items.map {|item|
+      WebItem.new url: item.url,
+                  title: item.title
+    }
+  end
+
+  def parse(items)
+    items.map do |item|
+      request = network.request(item.url)
+      request.meta = item
+      manager.queue request
+      request
+    end
+  end
+
+  def after_parse(requests)
+    requests.map {|request|
+      html = request.response_html
+      item = request.meta
+      item.description = html.og_description
+      unless html.og_image.blank?
+        item.image = html.og_image
+      end
+      item
+    }
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flute
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.1
 platform: ruby
 authors:
 - 'manjia
@@ -54,10 +54,67 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.0.1
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.2
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.0
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 一个简单的网络爬虫框架.
 email:
 - tywf91@gmail.com
-executables: []
+executables:
+- flute
 extensions: []
 extra_rdoc_files: []
 files:
@@ -71,9 +128,23 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- exe/flute
 - flute.gemspec
 - lib/flute.rb
+- lib/flute/document.rb
+- lib/flute/network.rb
+- lib/flute/request.rb
+- lib/flute/response.rb
+- lib/flute/spider_base.rb
 - lib/flute/version.rb
+- lib/template/Gemfile
+- lib/template/Rakefile
+- lib/template/items/base_item.rb
+- lib/template/items/rss_item.rb
+- lib/template/items/web_item.rb
+- lib/template/main.rb
+- lib/template/middlewares/rss_middleware.rb
+- lib/template/spiders/ifanr_spider.rb
 homepage: https://github.com/oxoooo/flute
 licenses:
 - MIT