kudzu-adapter-active_record 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b41da782783d660d5366be6d2c235d50722f8574
+  data.tar.gz: f41f69bf3555773358301b07e9b6832c36424a79
+SHA512:
+  metadata.gz: 1e3f5cc57bb822445c288c3a999614a18811ae3e835205b44295c8ac38b804327254436bb85236545822de1e097e20af35971dccb8cfd7bc09ffcf80f3ff86a5
+  data.tar.gz: d8626598ce2b61a69eb75f3d897c0b881847a2745bd36db63cb2443b6e5f24300c6e08562c219556c824c932f548ca65fece1303cf4be581771c71aa528eda26

data/README.md

@@ -0,0 +1,46 @@
+# Kudzu::Adapter::ActiveRecord
+
+ActiveRecord adapter for kudzu crawler.
+
+## Dependencies
+
+* kudzu 1.0+
+* activerecord 5.0+
+
+## Installation
+
+Add to your application's Gemfile:
+
+```ruby
+gem 'kudzu-adapter-active_record'
+```
+
+Then run:
+
+    $ bundle install
+
+Create migration files:
+
+    $ rails generate kudzu:adapter:active_record:migration
+
+Migrate into your application:
+
+    $ bundle exec rake db:migrate
+
+This migration creates following tables:
+
+* kudzu_pages
+* kudzu_contents
+* kudzu_links
+
+## Usage
+
+Basic usage is same as [kudzu gem](https://github.com/kanety/kudzu).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/kanety/kudzu-adapter-active_record. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/lib/generators/kudzu/adapter/active_record/migration_generator.rb

@@ -0,0 +1,28 @@
+require 'rails/generators'
+
+module Kudzu
+  module Adapter
+    module ActiveRecord
+      class MigrationGenerator < Rails::Generators::Base
+        source_root File.join(File.dirname(__FILE__), 'templates')
+
+        def create
+          @migration_version = migration_version
+          timestamp = Time.now.utc.strftime("%Y%m%d%H%M%S").to_i
+          ["create_kudzu_pages", "create_kudzu_contents", "create_kudzu_links"].each_with_index do |filename, i|
+            timestamp += i
+            template "#{filename}.rb.erb", "db/migrate/#{timestamp}_#{filename}.rb"
+          end
+        end
+
+        private
+
+        def migration_version
+          if ::ActiveRecord::VERSION::MAJOR >= 5
+            "[#{::ActiveRecord::VERSION::MAJOR}.#{::ActiveRecord::VERSION::MINOR}]"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/generators/kudzu/adapter/active_record/templates/create_kudzu_contents.rb.erb

@@ -0,0 +1,9 @@
+class CreateKudzuContents < ActiveRecord::Migration<%= @migration_version %>
+  def change
+    create_table :kudzu_contents do |t|
+      t.references :page
+      t.binary     :data
+      t.timestamps null: false
+    end
+  end
+end

data/lib/generators/kudzu/adapter/active_record/templates/create_kudzu_links.rb.erb

@@ -0,0 +1,16 @@
+class CreateKudzuLinks < ActiveRecord::Migration<%= @migration_version %>
+  def change
+    create_table :kudzu_links do |t|
+      t.text    :uuid
+      t.text    :url
+      t.text    :title
+      t.integer :state
+      t.integer :depth
+      t.timestamps null: false
+
+      t.index :uuid, length: 4
+      t.index :url, length: 32
+      t.index :state
+    end
+  end
+end

data/lib/generators/kudzu/adapter/active_record/templates/create_kudzu_pages.rb.erb

@@ -0,0 +1,24 @@
+class CreateKudzuPages < ActiveRecord::Migration<%= @migration_version %>
+  def change
+    create_table :kudzu_pages do |t|
+      t.text     :url
+      t.text     :title
+      t.integer  :status
+      t.text     :mime_type
+      t.integer  :size
+      t.text     :charset
+      t.text     :digest
+      t.text     :response_header
+      t.float    :response_time
+      t.text     :redirect_from
+      t.datetime :fetched_at
+      t.datetime :revised_at
+      t.integer  :revisit_interval
+      t.datetime :revisit_at
+      t.timestamps null: false
+
+      t.index :url, length: 32
+      t.index :digest, length: 6
+    end
+  end
+end

data/lib/kudzu/adapter/active_record.rb

@@ -0,0 +1,19 @@
+require 'activerecord-import'
+require 'kudzu'
+
+module Kudzu
+  module Adapter
+    module ActiveRecord
+    end
+  end
+end
+
+Kudzu.adapter = Kudzu::Adapter::ActiveRecord
+
+if defined? Railtie
+  ActiveSupport.on_load :active_record do
+    require_relative 'active_record/all'
+  end
+else
+  require_relative 'active_record/all'
+end

data/lib/kudzu/adapter/active_record/all.rb

@@ -0,0 +1,10 @@
+require_relative 'model/base'
+require_relative 'model/page'
+require_relative 'model/content'
+require_relative 'model/link'
+require_relative 'frontier'
+require_relative 'repository'
+
+Kudzu::Page = Kudzu::Adapter::ActiveRecord::Page
+Kudzu::Link = Kudzu::Adapter::ActiveRecord::Link
+Kudzu::Content = Kudzu::Adapter::ActiveRecord::Content

data/lib/kudzu/adapter/active_record/frontier.rb

@@ -0,0 +1,43 @@
+module Kudzu
+  module Adapter
+    module ActiveRecord
+      class Frontier
+        def initialize(uuid, config = {})
+          @uuid = uuid
+          @monitor = Monitor.new
+        end
+
+        def enqueue(links, depth: 0)
+          @monitor.synchronize do
+            links = filter_existing_urls(links)
+            Link.import(links)
+            links
+          end
+        end
+
+        def dequeue(limit: 1)
+          @monitor.synchronize do
+            links = Link.where(uuid: @uuid, state: 0).order(id: :asc).limit(limit).to_a
+            links.each do |link|
+              link.state = 1
+              link.save
+            end
+            links
+          end
+        end
+
+        def clear
+          Link.where(uuid: @uuid).delete_all
+        end
+
+        private
+
+        def filter_existing_urls(links)
+          urls = Array(links).map { |link| link.url }
+          existed_urls = Link.where(uuid: @uuid, url: urls).pluck(:url)
+          links.select { |link| !existed_urls.include?(link.url) }
+        end
+      end
+    end
+  end
+end

data/lib/kudzu/adapter/active_record/model/base.rb

@@ -0,0 +1,15 @@
+module Kudzu
+  module Adapter
+    module ActiveRecord
+      class Base < ::ActiveRecord::Base
+        self.abstract_class = true
+
+        class << self
+          def table_name_prefix
+            'kudzu_'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/kudzu/adapter/active_record/model/content.rb

@@ -0,0 +1,9 @@
+module Kudzu
+  module Adapter
+    module ActiveRecord
+      class Content < Base
+        belongs_to :page
+      end
+    end
+  end
+end

data/lib/kudzu/adapter/active_record/model/link.rb

@@ -0,0 +1,9 @@
+module Kudzu
+  module Adapter
+    module ActiveRecord
+      class Link < Base
+        include Kudzu::Adapter::Base::Link
+      end
+    end
+  end
+end

data/lib/kudzu/adapter/active_record/model/page.rb

@@ -0,0 +1,34 @@
+module Kudzu
+  module Adapter
+    module ActiveRecord
+      class Page < Base
+        include Kudzu::Adapter::Base::Page
+
+        has_one :content, dependent: :destroy
+
+        def response_header
+          if response_header_column_is_text?
+            JSON.parse(super)
+          else
+            super
+          end
+        end
+
+        def response_header=(val)
+          if response_header_column_is_text?
+            super(JSON.generate(val))
+          else
+            super
+          end
+        end
+
+        private
+
+        def response_header_column_is_text?
+          type = self.class.columns_hash["response_header"].type
+          type == :text || type == :string
+        end
+      end
+    end
+  end
+end

data/lib/kudzu/adapter/active_record/repository.rb

@@ -0,0 +1,23 @@
+module Kudzu
+  module Adapter
+    module ActiveRecord
+      class Repository
+        def find_by_url(url)
+          Page.where(url: url).first_or_initialize
+        end
+
+        def register(page)
+          if page.body
+            content = page.content || page.build_content
+            content.data = page.body
+          end
+          page.save
+        end
+
+        def delete(page)
+          page.destroy if page
+        end
+      end
+    end
+  end
+end

data/lib/kudzu/adapter/active_record/version.rb

@@ -0,0 +1,7 @@
+module Kudzu
+  module Adapter
+    module ActiveRecord
+      VERSION = '1.0.0'
+    end
+  end
+end

metadata

@@ -0,0 +1,199 @@
+--- !ruby/object:Gem::Specification
+name: kudzu-adapter-active_record
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Yoshikazu Kaneta
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-12-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: kudzu
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord-import
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: database_cleaner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ActiveRecord adapter for kudzu crawler
+email:
+- kaneta@sitebridge.co.jp
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- Rakefile
+- lib/generators/kudzu/adapter/active_record/migration_generator.rb
+- lib/generators/kudzu/adapter/active_record/templates/create_kudzu_contents.rb.erb
+- lib/generators/kudzu/adapter/active_record/templates/create_kudzu_links.rb.erb
+- lib/generators/kudzu/adapter/active_record/templates/create_kudzu_pages.rb.erb
+- lib/kudzu/adapter/active_record.rb
+- lib/kudzu/adapter/active_record/all.rb
+- lib/kudzu/adapter/active_record/frontier.rb
+- lib/kudzu/adapter/active_record/model/base.rb
+- lib/kudzu/adapter/active_record/model/content.rb
+- lib/kudzu/adapter/active_record/model/link.rb
+- lib/kudzu/adapter/active_record/model/page.rb
+- lib/kudzu/adapter/active_record/repository.rb
+- lib/kudzu/adapter/active_record/version.rb
+homepage: https://github.com/kanety/kudzu-adapter-active_record
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2.2
+signing_key: 
+specification_version: 4
+summary: ActiveRecord adapter for kudzu crawler
+test_files: []