kudzu-adapter-active_record 1.0.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: b41da782783d660d5366be6d2c235d50722f8574
-  data.tar.gz: f41f69bf3555773358301b07e9b6832c36424a79
+SHA256:
+  metadata.gz: 6070cb29d79bb57e5dd856f975ad58ae27d04ec1d5155a8ff93c65530ea3032c
+  data.tar.gz: a4f62c4f71e36dae0d5d9061443a606e54a3954220d0f94b059b9e6d38dd8978
 SHA512:
-  metadata.gz: 1e3f5cc57bb822445c288c3a999614a18811ae3e835205b44295c8ac38b804327254436bb85236545822de1e097e20af35971dccb8cfd7bc09ffcf80f3ff86a5
-  data.tar.gz: d8626598ce2b61a69eb75f3d897c0b881847a2745bd36db63cb2443b6e5f24300c6e08562c219556c824c932f548ca65fece1303cf4be581771c71aa528eda26
+  metadata.gz: 9ebf5a08ea9fc737fd342b4eadf443c01caafb0e14a43f78a60aac3cd0dc8a269d63635aee857fa59674ab20b7a41a464a486876e8b5a19c730e328af68a1c31
+  data.tar.gz: 2257f8e31654c5512a1b217b5edf9a657440ec73b2b909ac67b717b9cc7c3ebbf81166e7921c1536c3ca2f00c77601a3d8951a18a9789b36fc4213b5cc6592ac

data/README.md

@@ -4,7 +4,7 @@ ActiveRecord adapter for kudzu crawler.
 
 ## Dependencies
 
-* kudzu 1.0+
+* kudzu 1.1+
 * activerecord 5.0+
 
 ## Installation
@@ -30,7 +30,7 @@ Migrate into your application:
 This migration creates following tables:
 
 * kudzu_pages
-* kudzu_contents
+* kudzu_chunks
 * kudzu_links
 
 ## Usage

data/lib/generators/kudzu/adapter/active_record/migration_generator.rb

@@ -9,7 +9,7 @@ module Kudzu
         def create
           @migration_version = migration_version
           timestamp = Time.now.utc.strftime("%Y%m%d%H%M%S").to_i
-          ["create_kudzu_pages", "create_kudzu_contents", "create_kudzu_links"].each_with_index do |filename, i|
+          ["create_kudzu_pages", "create_kudzu_chunks", "create_kudzu_links"].each_with_index do |filename, i|
             timestamp += i
             template "#{filename}.rb.erb", "db/migrate/#{timestamp}_#{filename}.rb"
           end

data/lib/generators/kudzu/adapter/active_record/templates/create_kudzu_chunks.rb.erb

@@ -0,0 +1,9 @@
+class CreateKudzuChunks < ActiveRecord::Migration<%= @migration_version %>
+  def change
+    create_table :kudzu_chunks do |t|
+      t.references :page
+      t.binary     :data
+      t.timestamps null: false
+    end
+  end
+end

data/lib/generators/kudzu/adapter/active_record/templates/create_kudzu_pages.rb.erb

@@ -13,8 +13,6 @@ class CreateKudzuPages < ActiveRecord::Migration<%= @migration_version %>
       t.text     :redirect_from
       t.datetime :fetched_at
       t.datetime :revised_at
-      t.integer  :revisit_interval
-      t.datetime :revisit_at
       t.timestamps null: false
 
       t.index :url, length: 32

data/lib/kudzu/adapter/active_record/all.rb

@@ -1,10 +1,6 @@
 require_relative 'model/base'
 require_relative 'model/page'
-require_relative 'model/content'
+require_relative 'model/chunk'
 require_relative 'model/link'
 require_relative 'frontier'
 require_relative 'repository'
-
-Kudzu::Page = Kudzu::Adapter::ActiveRecord::Page
-Kudzu::Link = Kudzu::Adapter::ActiveRecord::Link
-Kudzu::Content = Kudzu::Adapter::ActiveRecord::Content

data/lib/kudzu/adapter/active_record/frontier.rb

@@ -10,7 +10,7 @@ module Kudzu
         def enqueue(links, depth: 0)
           @monitor.synchronize do
             links = filter_existing_urls(links)
-            Link.import(links)
+            Link.bulk_import(links)
             links
           end
         end

data/lib/kudzu/adapter/active_record/model/chunk.rb

@@ -0,0 +1,13 @@
+module Kudzu
+  module Adapter
+    module ActiveRecord
+      class Chunk < Base
+        belongs_to :page
+
+        scope :select_without_data, -> { select(column_names - %w(data)) }
+      end
+    end
+  end
+
+  Chunk = Adapter::ActiveRecord::Chunk
+end

data/lib/kudzu/adapter/active_record/model/link.rb

@@ -2,8 +2,10 @@ module Kudzu
   module Adapter
     module ActiveRecord
       class Link < Base
-        include Kudzu::Adapter::Base::Link
+        include Kudzu::Model::Link
       end
     end
   end
+
+  Link = Adapter::ActiveRecord::Link
 end

data/lib/kudzu/adapter/active_record/model/page.rb

@@ -2,9 +2,9 @@ module Kudzu
   module Adapter
     module ActiveRecord
       class Page < Base
-        include Kudzu::Adapter::Base::Page
+        include Kudzu::Model::Page
 
-        has_one :content, dependent: :destroy
+        has_many :chunks, -> { order(id: :asc) }, dependent: :delete_all
 
         def response_header
           if response_header_column_is_text?
@@ -22,6 +22,10 @@ module Kudzu
           end
         end
 
+        def data
+          chunks.pluck(:data).join
+        end
+
         private
 
         def response_header_column_is_text?
@@ -31,4 +35,6 @@ module Kudzu
       end
     end
   end
+
+  Page = Adapter::ActiveRecord::Page
 end

data/lib/kudzu/adapter/active_record/railtie.rb

@@ -0,0 +1,18 @@
+module Kudzu
+  module Adapter
+    module ActiveRecord
+      class Railtie < Rails::Railtie
+        ActiveSupport.on_load :active_record do
+          require_relative 'all'
+        end
+
+        config.after_initialize do
+          Dir.glob(Rails.root + 'app/decorators/kudzu/**/*_decorator*.rb').each do |c|
+            require_dependency(c)
+          end
+          Kudzu.logger = Rails.logger
+        end
+      end
+    end
+  end
+end

data/lib/kudzu/adapter/active_record/repository.rb

@@ -3,20 +3,45 @@ module Kudzu
     module ActiveRecord
       class Repository
         def find_by_url(url)
-          Page.where(url: url).first_or_initialize
+          Page.find_or_initialize_by(url: url)
         end
 
         def register(page)
-          if page.body
-            content = page.content || page.build_content
-            content.data = page.body
+          ActiveRecord::Base.transaction do
+            page.save
+            save_chunks(page) if page.body
           end
-          page.save
         end
 
         def delete(page)
           page.destroy if page
         end
+
+        private
+
+        def save_chunks(page)
+          chunk_num = 0
+          each_chunk(page.body, ActiveRecord.chunk_size) do |chunked, i|
+            chunk = page.chunks.select_without_data.offset(i).limit(1).first_or_initialize
+            chunk.data = chunked
+            chunk.save
+            chunk_num = i + 1
+          end
+
+          if page.chunks.count - chunk_num > 0
+            page.chunks.select_without_data.offset(chunk_num).each(&:delete)
+          end
+        end
+
+        def each_chunk(body, size)
+          pos = 0
+          i = 0
+          while (chunked = body.byteslice(pos, size))
+            yield chunked, i
+            pos += size
+            i += 1
+          end
+        end
       end
     end
   end

data/lib/kudzu/adapter/active_record/version.rb

@@ -1,7 +1,7 @@
 module Kudzu
   module Adapter
     module ActiveRecord
-      VERSION = '1.0.0'
+      VERSION = '1.2.0'
     end
   end
 end

data/lib/kudzu/adapter/active_record.rb

@@ -1,19 +1,28 @@
 require 'activerecord-import'
 require 'kudzu'
 
+if defined? Rails
+  require_relative 'active_record/railtie'
+else
+  require_relative 'active_record/all'
+end
+
 module Kudzu
   module Adapter
     module ActiveRecord
+      class << self
+        @@chunk_size = 5*(1024**2)
+
+        def chunk_size
+          @@chunk_size
+        end
+  
+        def chunk_size=(val)
+          @@chunk_size = val
+        end
+      end
     end
   end
 end
 
 Kudzu.adapter = Kudzu::Adapter::ActiveRecord
-
-if defined? Railtie
-  ActiveSupport.on_load :active_record do
-    require_relative 'active_record/all'
-  end
-else
-  require_relative 'active_record/all'
-end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kudzu-adapter-active_record
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Yoshikazu Kaneta
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-12-20 00:00:00.000000000 Z
+date: 2024-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: kudzu
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
@@ -67,7 +67,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: sqlite3
+  name: webrick
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -81,7 +81,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rspec-rails
+  name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -95,7 +95,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: simplecov
+  name: mysql2
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -109,7 +109,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pry-rails
+  name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -123,7 +123,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pry-byebug
+  name: rspec-rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -137,7 +137,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: database_cleaner
+  name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -160,23 +160,24 @@ files:
 - README.md
 - Rakefile
 - lib/generators/kudzu/adapter/active_record/migration_generator.rb
-- lib/generators/kudzu/adapter/active_record/templates/create_kudzu_contents.rb.erb
+- lib/generators/kudzu/adapter/active_record/templates/create_kudzu_chunks.rb.erb
 - lib/generators/kudzu/adapter/active_record/templates/create_kudzu_links.rb.erb
 - lib/generators/kudzu/adapter/active_record/templates/create_kudzu_pages.rb.erb
 - lib/kudzu/adapter/active_record.rb
 - lib/kudzu/adapter/active_record/all.rb
 - lib/kudzu/adapter/active_record/frontier.rb
 - lib/kudzu/adapter/active_record/model/base.rb
-- lib/kudzu/adapter/active_record/model/content.rb
+- lib/kudzu/adapter/active_record/model/chunk.rb
 - lib/kudzu/adapter/active_record/model/link.rb
 - lib/kudzu/adapter/active_record/model/page.rb
+- lib/kudzu/adapter/active_record/railtie.rb
 - lib/kudzu/adapter/active_record/repository.rb
 - lib/kudzu/adapter/active_record/version.rb
 homepage: https://github.com/kanety/kudzu-adapter-active_record
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -191,9 +192,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2.2
-signing_key: 
+rubygems_version: 3.3.3
+signing_key:
 specification_version: 4
 summary: ActiveRecord adapter for kudzu crawler
 test_files: []

data/lib/generators/kudzu/adapter/active_record/templates/create_kudzu_contents.rb.erb

@@ -1,9 +0,0 @@
-class CreateKudzuContents < ActiveRecord::Migration<%= @migration_version %>
-  def change
-    create_table :kudzu_contents do |t|
-      t.references :page
-      t.binary     :data
-      t.timestamps null: false
-    end
-  end
-end

data/lib/kudzu/adapter/active_record/model/content.rb

@@ -1,9 +0,0 @@
-module Kudzu
-  module Adapter
-    module ActiveRecord
-      class Content < Base
-        belongs_to :page
-      end
-    end
-  end
-end