miteru 1.0.2 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2217ef5ac9ed2060203df9f5edebbac779fbebc53e4418f54de6d4b080ee81fc
-  data.tar.gz: 45c934a406c619b1d1cc9d03a00103ae2b732460ca70877953aaac2d36d3b0e0
+  metadata.gz: 050c27599e75745a7c215f08b7ed190b43c70388d974a68945702eefdb25b7c2
+  data.tar.gz: 64c7429a4178febf6984fe3d79d3970781634971405c5d9fdb0748c61b663a32
 SHA512:
-  metadata.gz: 03ebfc42660abedfa61a7e931e0d863b03fc50d615cfe0c81db10c0473ae8e0908619c5bcbc656e48784d40b5ccc5b98d616efe467d2a68ecca1bb57415bacfb
-  data.tar.gz: a74accbf1d3949e8211792e02c68e6e6911acbe2cc0cd113459eb812bad29ae0dc4c434f56009a1e49fa255d58bfc5485b9cbe8e3d6e936fef68e23d671d7d86
+  metadata.gz: ff780a0db3fafdded94261c38272dc0537462ae195f646cceafc2223cd62e61fb8d809ee172f953344314a93f9e6f746b9d3b6b66efdf9b48cda2c5d8c645eb7
+  data.tar.gz: c55dfad5120175ebf43bd01a30b42517a9a5bfd6fddd12d8c6ad07acd3ae02bba67a1bc9101d26fab74406f374f16fbeb3f51e791c678dce749a403e9477f82a

data/lib/miteru/crawler.rb

@@ -14,8 +14,8 @@ module Miteru
       @notifier = Notifier.new
     end
 
-    def crawl(url)
-      website = Website.new(url)
+    def crawl(entry)
+      website = Website.new(entry.url, entry.source)
       downloader.download_kits(website.kits) if website.has_kits? && auto_download?
       notify(website) if website.has_kits? || verbose?
     rescue OpenSSL::SSL::SSLError, HTTP::Error, Addressable::URI::InvalidURIError => _e
@@ -23,11 +23,11 @@ module Miteru
     end
 
     def execute
-      suspicious_urls = feeds.suspicious_urls
-      puts "Loaded #{suspicious_urls.length} URLs to crawl. (crawling in #{threads} threads)" if verbose?
+      suspicious_entries = feeds.suspicious_entries
+      puts "Loaded #{suspicious_entries.length} URLs to crawl. (crawling in #{threads} threads)" if verbose?
 
-      Parallel.each(suspicious_urls, in_threads: threads) do |url|
-        crawl url
+      Parallel.each(suspicious_entries, in_threads: threads) do |entry|
+        crawl entry
       end
     end
 

data/lib/miteru/database.rb

@@ -19,6 +19,12 @@ class InitialSchema < ActiveRecord::Migration[6.1]
   end
 end
 
+class V11Schema < ActiveRecord::Migration[6.1]
+  def change
+    add_column :records, :source, :string, if_not_exists: true
+  end
+end
+
 def adapter
   return "postgresql" if Miteru.configuration.database.start_with?("postgresql://", "postgres://")
   return "mysql2" if Miteru.configuration.database.start_with?("mysql2://")
@@ -44,6 +50,7 @@ module Miteru
         ActiveRecord::Migration.verbose = false
 
         InitialSchema.migrate(:up)
+        V11Schema.migrate(:up)
       rescue StandardError => _e
         # Do nothing
       end
@@ -57,6 +64,7 @@ module Miteru
         return unless ActiveRecord::Base.connected?
 
         InitialSchema.migrate(:down)
+        V11Schema.migrate(:down)
       end
     end
   end

data/lib/miteru/feeds/feed.rb

@@ -3,10 +3,43 @@
 module Miteru
   class Feeds
     class Feed
+      include Mixins::URL
+
+      def source
+        @source ||= self.class.to_s.split("::").last
+      end
+
+      #
+      # Return URLs
+      #
+      # @return [Array<String>] URLs
+      #
       def urls
         raise NotImplementedError, "You must implement #{self.class}##{__method__}"
       end
 
+      #
+      # Return entries
+      #
+      # @return [Array<Miteru::Entry>]
+      #
+      def entries
+        breakdowend_urls.map do |url|
+          Entry.new(url, source)
+        end
+      end
+
+      #
+      # Return breakdowned URLs
+      #
+      # @return [Array<String>] Breakdowned URLs
+      #
+      def breakdowend_urls
+        urls.select { |url| url.start_with?("http://", "https://") }.map do |url|
+          breakdown(url, Miteru.configuration.directory_traveling?)
+        end.flatten.uniq
+      end
+
       private
 
       def get(url)

data/lib/miteru/feeds.rb

@@ -8,6 +8,18 @@ require_relative "./feeds/urlscan"
 require_relative "./feeds/urlscan_pro"
 
 module Miteru
+  class Entry
+    # @return [String]
+    attr_reader :url
+    # @return [String]
+    attr_reader :source
+
+    def initialize(url, source)
+      @url = url
+      @source = source
+    end
+  end
+
   class Feeds
     IGNORE_EXTENSIONS = %w[.htm .html .php .asp .aspx .exe .txt].freeze
 
@@ -21,43 +33,13 @@ module Miteru
       ].compact
     end
 
-    def directory_traveling?
-      Miteru.configuration.directory_traveling?
-    end
-
-    def suspicious_urls
-      @suspicious_urls ||= [].tap do |arr|
-        urls = @feeds.map do |feed|
-          feed.urls.select { |url| url.start_with?("http://", "https://") }
-        end.flatten.uniq
-
-        urls.map { |url| breakdown(url) }.flatten.uniq.sort.each { |url| arr << url }
-      end
-    end
-
-    def breakdown(url)
-      begin
-        uri = URI.parse(url)
-      rescue URI::InvalidURIError => _e
-        return []
-      end
-
-      base = "#{uri.scheme}://#{uri.hostname}"
-      return [base] unless directory_traveling?
-
-      segments = uri.path.split("/")
-      return [base] if segments.length.zero?
-
-      urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join("/")}" }
-
-      urls.reject do |breakdowned_url|
-        # Reject a url which ends with specific extension names
-        invalid_extension? breakdowned_url
-      end
-    end
-
-    def invalid_extension?(url)
-      IGNORE_EXTENSIONS.any? { |ext| url.end_with? ext }
+    #
+    # Returns a list of suspicious entries
+    #
+    # @return [Array<Entry>]
+    #
+    def suspicious_entries
+      @suspicious_entries ||= @feeds.map(&:entries).flatten.uniq(&:url)
     end
   end
 end

data/lib/miteru/kit.rb

@@ -9,10 +9,27 @@ module Miteru
     VALID_EXTENSIONS = Miteru.configuration.valid_extensions
     VALID_MIME_TYPES = Miteru.configuration.valid_mime_types
 
-    attr_reader :url, :status, :content_length, :mime_type, :headers
+    # @return [String]
+    attr_reader :url
 
-    def initialize(url)
+    # @return [String]
+    attr_reader :source
+
+    # @return [Integer, nil]
+    attr_reader :status
+
+    # @return [Integer, nil]
+    attr_reader :content_length
+
+    # @return [String, nil]
+    attr_reader :mime_type
+
+    # @return [Hash, nil]
+    attr_reader :headers
+
+    def initialize(url, source)
       @url = url
+      @source = source
 
       @content_length = nil
       @mime_type = nil

data/lib/miteru/mixin.rb

@@ -0,0 +1,47 @@
+module Miteru
+  module Mixins
+    module URL
+      IGNORE_EXTENSIONS = %w[.htm .html .php .asp .aspx .exe .txt].freeze
+
+      #
+      # Validate extension of a URL
+      #
+      # @param [String] url
+      #
+      # @return [Boolean]
+      #
+      def invalid_extension?(url)
+        IGNORE_EXTENSIONS.any? { |ext| url.end_with? ext }
+      end
+
+      #
+      # Breakdown a URL into URLs
+      #
+      # @param [String] url
+      # @param [Boolean] enable_directory_traveling
+      #
+      # @return [Array<String>]
+      #
+      def breakdown(url, enable_directory_traveling)
+        begin
+          uri = URI.parse(url)
+        rescue URI::InvalidURIError => _e
+          return []
+        end
+
+        base = "#{uri.scheme}://#{uri.hostname}"
+        return [base] unless enable_directory_traveling
+
+        segments = uri.path.split("/")
+        return [base] if segments.length.zero?
+
+        urls = (0...segments.length).map { |idx| "#{base}#{segments[0..idx].join("/")}" }
+
+        urls.reject do |breakdowned_url|
+          # Reject a url which ends with specific extension names
+          invalid_extension? breakdowned_url
+        end
+      end
+    end
+  end
+end

data/lib/miteru/record.rb

@@ -30,6 +30,7 @@ module Miteru
       def create_by_kit_and_hash(kit, hash)
         record = new(
           hash: hash,
+          source: kit.source,
           hostname: kit.hostname,
           url: kit.decoded_url,
           headers: kit.headers,

data/lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "1.0.2"
+  VERSION = "1.1.0"
 end

data/lib/miteru/website.rb

@@ -6,10 +6,15 @@ module Miteru
   class Website
     VALID_EXTENSIONS = Miteru.configuration.valid_extensions
 
+    # @return [String]
     attr_reader :url
 
-    def initialize(url)
+    # @return [String]
+    attr_reader :source
+
+    def initialize(url, source)
       @url = url
+      @source = source
     end
 
     def title
@@ -18,7 +23,7 @@ module Miteru
 
     def kits
       @kits ||= links.filter_map do |link|
-        kit = Kit.new(link)
+        kit = Kit.new(link, source)
         kit.valid? ? kit : nil
       end
     end

data/lib/miteru.rb

@@ -7,6 +7,8 @@ require "miteru/database"
 
 require "miteru/record"
 
+require "miteru/mixin"
+
 require "miteru/error"
 require "miteru/http_client"
 require "miteru/kit"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miteru
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.1.0
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-09-10 00:00:00.000000000 Z
+date: 2021-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -370,6 +370,7 @@ files:
 - lib/miteru/feeds/urlscan_pro.rb
 - lib/miteru/http_client.rb
 - lib/miteru/kit.rb
+- lib/miteru/mixin.rb
 - lib/miteru/notifier.rb
 - lib/miteru/record.rb
 - lib/miteru/version.rb