fushin 0.1.0

data/lib/fushin/erros.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Fushin
+  class NoMachingPostsError < StandardError; end
+end

data/lib/fushin/item.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Fushin
+  class Item
+    attr_reader :title, :link
+
+    def initialize(title:, link:)
+      @title = title
+      @link = link
+    end
+
+    def post
+      @post ||= [].tap do |out|
+        case link
+        when /jugem\.jp/
+          out << Posts::Jugem.new(link)
+        when /kikey\.net/
+          out << Posts::Kikey.new(link)
+        when /seesaa\.net/
+          otu << Posts::Seesaa.new(link)
+        when /shinobi\.jp/
+          out << Posts::Shinobi.new(link)
+        when /teacup\.com/
+          out << Posts::Teacup.new(link)
+        else
+          raise NoMachingPostsError
+        end
+      end.first
+    end
+  end
+end

data/lib/fushin/models/btc.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Fushin
+  module Models
+    class BTC < Model
+      attr_reader :address
+      def initialize(address)
+        @address = address
+      end
+
+      def title
+        "BTC: #{address}"
+      end
+
+      def blockchain_link
+        "https://www.blockchain.com/btc/address/#{address}"
+      end
+
+      def to_attachements
+        [
+          {
+            fallback: "blockchain.com link",
+            title: title,
+            title_link: blockchain_link,
+            footer: "blockchain.com",
+            footer_icon: "http://www.google.com/s2/favicons?domain=blockchain.com"
+          }
+        ]
+      end
+    end
+  end
+end

data/lib/fushin/models/model.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Fushin
+  module Models
+    class Model
+      def title
+        raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+      end
+
+      def to_attachements
+        raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+      end
+    end
+  end
+end

data/lib/fushin/models/website.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "digest/sha2"
+require "uri"
+
+module Fushin
+  module Models
+    class Website < Model
+      attr_reader :url
+      def initialize(url)
+        @url = url
+      end
+
+      def uri
+        @uri ||= URI(url)
+      end
+
+      def domain
+        uri.hostname
+      end
+
+      def urlscan_link
+        "https://urlscan.io/domain/#{domain}"
+      end
+
+      def url_for_vt
+        uri.path.empty? ? "#{url}/" : url
+      end
+
+      def vt_link
+        "https://www.virustotal.com/#/url/#{Digest::SHA256.hexdigest(url_for_vt)}"
+      end
+
+      def to_attachements
+        [
+          {
+            fallback: "virustotal.com link",
+            title: "VT: #{url}",
+            title_link: vt_link,
+            footer: "virustotal.com",
+            footer_icon: "http://www.google.com/s2/favicons?domain=virustotal.com"
+          },
+          {
+            fallback: "urlscan.io link",
+            title: "urlscan.io: #{domain}",
+            title_link: urlscan_link,
+            footer: "urlscan.io",
+            footer_icon: "http://www.google.com/s2/favicons?domain=urlscan.io"
+          },
+        ]
+      end
+    end
+  end
+end

data/lib/fushin/monitor.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Fushin
+  class Monitor
+    attr_reader :rss
+    def initialize
+      @rss = RSS.new
+    end
+
+    def check
+      rss.items.each do |item|
+        next if Cache.cached?(item.link)
+
+        attachements = [].tap do |out|
+          out << item.post.btcs.map(&:to_attachements)
+          out << item.post.urls.map(&:to_attachements)
+          out << item.post.links.map(&:to_attachements)
+        end.flatten
+        attachements << { text: "IoC is not found." } if attachements.empty?
+        Notifier.notify("#{item.title} (#{item.link})", attachements)
+
+        Cache.save(item.link, true)
+      rescue StandardError => e
+        puts "#{e.class} (#{e}) is happened during processing #{item.link}"
+      end
+    end
+
+    def self.check
+      new.check
+    end
+  end
+end

data/lib/fushin/notifier.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "slack/incoming/webhooks"
+
+module Fushin
+  class Notifier
+    def notify(title, attachments = [])
+      if slack_webhook_url?
+        slack = Slack::Incoming::Webhooks.new(slack_webhook_url, channel: slack_channel)
+        slack.post title, attachments: attachments
+      else
+        puts title
+        attachments.each do |attachment|
+          puts "#{attachment.dig(:title)} (#{attachment.dig(:title_link)})"
+        end
+      end
+    end
+
+    def slack_webhook_url
+      ENV.fetch "SLACK_WEBHOOK_URL"
+    end
+
+    def slack_channel
+      ENV.fetch "SLACK_CHANNEL", "#general"
+    end
+
+    def slack_webhook_url?
+      ENV.key? "SLACK_WEBHOOK_URL"
+    end
+
+    def self.notify(title, text)
+      new.notify(title, text)
+    end
+  end
+end

data/lib/fushin/posts/jugem.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Fushin
+  module Posts
+    class Jugem < Post
+      def main_selector
+        "div.entry_body"
+      end
+
+      def main_cleanup_selectors
+        %w(script iframe .service_button)
+      end
+    end
+  end
+end

data/lib/fushin/posts/kikey.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Fushin
+  module Posts
+    class Kikey < Post
+      def main_selector
+        "#categoryBlog > div.entryBody"
+      end
+
+      def main_cleanup_selectors
+        []
+      end
+    end
+  end
+end

data/lib/fushin/posts/post.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require "charlock_holmes"
+require "http"
+require "oga"
+require "uri"
+require "yaml"
+require "url_regex"
+
+module Fushin
+  module Posts
+    class Post
+      attr_reader :url
+
+      def initialize(url)
+        @url = url
+      end
+
+      def main
+        @main ||= [].tap do |out|
+          body = doc.at_css(main_selector)
+
+          main_cleanup_selectors.each do |selector|
+            body.css(selector).each(&:remove)
+          end
+
+          out << body
+        end.first
+      end
+
+      def main_text
+        @main_text ||= [].tap do |out|
+          detection = CharlockHolmes::EncodingDetector.detect(main.text)
+          out << CharlockHolmes::Converter.convert(main.text, detection[:encoding], "UTF-8")
+        end.first
+      end
+
+      def btcs
+        @btcs ||= main_text.scan(/\b[13][a-km-zA-HJ-NP-Z0-9]{26,33}\b/).uniq.map do |address|
+          Models::BTC.new(address)
+        end
+      end
+
+      def urls
+        @urls ||= main_text.scan(UrlRegex.get(scheme_required: true, mode: :parsing)).uniq.map do |url|
+          next if whitelisted_domain?(url)
+
+          Models::Website.new(url)
+        end.compact
+      end
+
+      def links
+        @links ||= main.css("a").map { |a| a.get("href") }.compact.uniq.map do |url|
+          next if whitelisted_domain?(url)
+
+          Models::Website.new(url)
+        end.compact
+      end
+
+      private
+
+      def main_selector
+        raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+      end
+
+      def main_cleanup_selectors
+        raise NotImplementedError, "You must implement #{self.class}##{__method__}"
+      end
+
+      def whitelisted_domain?(url)
+        uri = URI(url)
+        whitelisted_domains.any? { |domain| uri.hostname.end_with? domain }
+      end
+
+      def whitelisted_domains
+        @whitelisted_domains ||= YAML.safe_load(
+          File.read(File.expand_path("./../config/whitelisted_domains.yml", __dir__))
+        )
+      end
+
+      def doc
+        @doc ||= Oga.parse_html(body)
+      end
+
+      def body
+        res = HTTP.get(url)
+        return nil unless res.code == 200
+
+        res.body.to_s
+      end
+    end
+  end
+end

data/lib/fushin/posts/seesaa.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Fushin
+  module Posts
+    class Seesaa < Post
+      def main_selector
+        "#content > div.blog > div > div.text"
+      end
+
+      def main_cleanup_selectors
+        %w(script iframe .listCategoryArticle)
+      end
+    end
+  end
+end

data/lib/fushin/posts/shinobi.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Fushin
+  module Posts
+    class Shinobi < Post
+      def main_selector
+        "#primary > div.inner > div > div:nth-child(3)"
+      end
+
+      def main_cleanup_selectors
+        %w(script)
+      end
+    end
+  end
+end

data/lib/fushin/posts/teacup.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Fushin
+  module Posts
+    class Teacup < Post
+      def main_selector
+        "#tcmainlay > div.postdate > div > div.postbody"
+      end
+
+      def main_cleanup_selectors
+        %w(script .tagword)
+      end
+    end
+  end
+end

data/lib/fushin/rss.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "http"
+require "rss"
+require "uri"
+
+module Fushin
+  class RSS
+    BASE_URL = "https://www.inoreader.com/stream/user/1006141524/tag/%E4%B8%8D%E5%AF%A9%E3%83%A1%E3%83%BC%E3%83%AB"
+
+    def feed
+      @feed ||= ::RSS::Parser.parse(body)
+    end
+
+    def items
+      feed.items.map do |item|
+        Item.new(title: item.title, link: item.link)
+      end
+    end
+
+    def body
+      res = HTTP.get(BASE_URL)
+      return nil unless res.code == 200
+
+      res.body.to_s
+    end
+  end
+end

data/lib/fushin/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Fushin
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,256 @@
+--- !ruby/object:Gem::Specification
+name: fushin
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Manabu Niseki
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2019-01-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.3'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.8'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+- !ruby/object:Gem::Dependency
+  name: charlock_holmes
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: lightly
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: oga
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.15'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.15'
+- !ruby/object:Gem::Dependency
+  name: rss
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: slack-incoming-webhooks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: url_regex
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
+description: A malicious blog posts checker
+email:
+- manabu.niseki@gmail.com
+executables:
+- fushin
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/fushin
+- fushin.gemspec
+- lib/fushin.rb
+- lib/fushin/cache.rb
+- lib/fushin/config/whitelisted_domains.yml
+- lib/fushin/erros.rb
+- lib/fushin/item.rb
+- lib/fushin/models/btc.rb
+- lib/fushin/models/model.rb
+- lib/fushin/models/website.rb
+- lib/fushin/monitor.rb
+- lib/fushin/notifier.rb
+- lib/fushin/posts/jugem.rb
+- lib/fushin/posts/kikey.rb
+- lib/fushin/posts/post.rb
+- lib/fushin/posts/seesaa.rb
+- lib/fushin/posts/shinobi.rb
+- lib/fushin/posts/teacup.rb
+- lib/fushin/rss.rb
+- lib/fushin/version.rb
+homepage: https://github.com/ninoseki/fushin
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.2
+signing_key: 
+specification_version: 4
+summary: A malicious blog posts checker
+test_files: []