rails_spider 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: d9c9f8d63d2d08f317bf12639ca4a407650632c8060cfe86bdffbb29eb120b03
+  data.tar.gz: 5d83cf89db880b21255990b1c14b9eb545cfd4e92c1ab918df5a65eb6142219f
+SHA512:
+  metadata.gz: 6be8dcf45f11b214314586b4035ddace9e41bb823d4c94a4d01797b62b0525b975df392560926d2bab30e392d1481e2cfec61d49db9f5938f38dca9c24564f17
+  data.tar.gz: 1058ed875f9ecd7cde8029af7b2811b619dfc77c01ac526ce2ce92ff2800d452fe42bf47663f5dda7e09bded946775e2d64295a8f0b36c7b5d6e901b59bd3ff4

data/LICENSE

@@ -0,0 +1,165 @@
+                   GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2018 Mingyuan Qin.
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.

data/README.md

@@ -0,0 +1,33 @@
+# RailsSpider
+Short description and motivation.
+
+## Usage
+How to use my plugin.
+日志在development.log，错误日志在error.log中
+
+代理IP，存在在文件proxy/proxylists.txt，如要更新代理IP，执行rake rake proxy:proxy_youdaili，会更新proxy/proxylists.txt
+
+
+
+## Installation
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'rails_spider'
+```
+
+And then execute:
+```bash
+$ bundle
+```
+
+Or install it yourself as:
+```bash
+$ gem install rails_spider
+```
+
+## Contributing
+Contribution directions go here.
+
+## License
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/app/assets/config/the_spider_manifest.js

@@ -0,0 +1,4 @@
+//= link_directory ../javascripts/rails_spider .js
+//= link_directory ../stylesheets/rails_spider .css
+//= link rails_spider/application.js
+//= link rails_spider/application.css

data/app/assets/javascripts/the_spider/application.js

@@ -0,0 +1 @@
+//= require rails-ujs

data/app/assets/stylesheets/the_spider/application.css

@@ -0,0 +1,4 @@
+/*
+*= require semantic
+*= require_self
+*/

data/app/controllers/the_spider/application_controller.rb

@@ -0,0 +1,9 @@
+module RailsSpider
+  class ApplicationController < ActionController::Base
+    protect_from_forgery with: :exception
+    default_form_builder 'RailsSpiderBuilder' do |config|
+
+    end
+
+  end
+end

data/app/controllers/the_spider/locals_controller.rb

@@ -0,0 +1,62 @@
+require_dependency "rails_spider/application_controller"
+
+module RailsSpider
+  class LocalsController < ApplicationController
+    before_action :set_local, only: [:show, :edit, :update, :destroy]
+
+    # GET /locals
+    def index
+      @locals = Local.page(params[:page])
+    end
+
+    # GET /locals/1
+    def show
+    end
+
+    # GET /locals/new
+    def new
+      @local = Local.new
+    end
+
+    # GET /locals/1/edit
+    def edit
+    end
+
+    # POST /locals
+    def create
+      @local = Local.new(local_params)
+
+      if @local.save
+        redirect_to @local, notice: 'Local was successfully created.'
+      else
+        render :new
+      end
+    end
+
+    # PATCH/PUT /locals/1
+    def update
+      if @local.update(local_params)
+        redirect_to @local, notice: 'Local was successfully updated.'
+      else
+        render :edit
+      end
+    end
+
+    # DELETE /locals/1
+    def destroy
+      @local.destroy
+      redirect_to locals_url, notice: 'Local was successfully destroyed.'
+    end
+
+    private
+      # Use callbacks to share common setup or constraints between actions.
+      def set_local
+        @local = Local.find(params[:id])
+      end
+
+      # Only allow a trusted parameter "white list" through.
+      def local_params
+        params.fetch(:local, {})
+      end
+  end
+end

data/app/controllers/the_spider/works_controller.rb

@@ -0,0 +1,60 @@
+require_dependency 'rails_spider/application_controller'
+module RailsSpider
+  class WorksController < ApplicationController
+    before_action :set_work, only: [:show, :edit, :update, :run, :destroy]
+
+    def index
+      @works = Work.page(params[:page])
+    end
+
+    def show
+    end
+
+    def new
+      @work = Work.new
+    end
+
+    def create
+      @work = Work.new(work_params)
+
+      if @work.save
+        redirect_to @work, notice: 'Work was successfully created.'
+      else
+        render :new
+      end
+    end
+
+    def edit
+    end
+
+    def update
+      if @work.update(work_params)
+        redirect_to @work, notice: 'Work was successfully updated.'
+      else
+        render :edit
+      end
+    end
+
+    def run
+      WorkJob.perform_later(@work.id)
+    end
+
+    def parser
+      ParserJob.perform_later(@work.id)
+    end
+
+    def destroy
+      @work.destroy
+      redirect_to works_url, notice: 'Work was successfully destroyed.'
+    end
+
+    private
+    def set_work
+      @work = Work.find(params[:id])
+    end
+
+    def work_params
+      params.fetch(:work, {}).permit(:name, :parser_name, :host, :list_path, :item_path, :page_params)
+    end
+  end
+end

data/app/helpers/the_spider/application_helper.rb

@@ -0,0 +1,4 @@
+module RailsSpider
+  module ApplicationHelper
+  end
+end

data/app/helpers/the_spider/locals_helper.rb

@@ -0,0 +1,4 @@
+module RailsSpider
+  module LocalsHelper
+  end
+end

data/app/helpers/the_spider/works_helper.rb

@@ -0,0 +1,4 @@
+module RailsSpider
+  module WorksHelper
+  end
+end

data/app/jobs/the_spider/application_job.rb

@@ -0,0 +1,4 @@
+module RailsSpider
+  class ApplicationJob < ActiveJob::Base
+  end
+end

data/app/jobs/the_spider/parser_job.rb

@@ -0,0 +1,11 @@
+module RailsSpider
+  class ParserJob < ApplicationJob
+    queue_as :default
+
+    def perform(work_id)
+      work = Work.find work_id
+      work.parse
+    end
+
+  end
+end

data/app/jobs/the_spider/work_job.rb

@@ -0,0 +1,11 @@
+module RailsSpider
+  class WorkJob < ApplicationJob
+    queue_as :default
+
+    def perform(work_id)
+      work = Work.find work_id
+      work.resource.run
+    end
+
+  end
+end

data/app/mailers/the_spider/application_mailer.rb

@@ -0,0 +1,6 @@
+module RailsSpider
+  class ApplicationMailer < ActionMailer::Base
+    default from: 'from@example.com'
+    layout 'mailer'
+  end
+end

data/app/models/rails_spider/application_record.rb

@@ -0,0 +1,5 @@
+module RailsSpider
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

data/app/models/rails_spider/cookie.rb

@@ -0,0 +1,9 @@
+module RailsSpider
+  class Cookie < ApplicationRecord
+    attribute :name, type: String
+    attribute :password, type: String
+    attribute :domain, type: String
+    attribute :value, type: String
+
+  end
+end

data/app/models/rails_spider/failed_url.rb

@@ -0,0 +1,7 @@
+module RailsSpider
+  class FailedUrl < ApplicationRecord
+    attribute :url, type: String
+    attribute :source, type: String
+    attribute :flag, type: String
+  end
+end