fushin 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4a3216a5de562191b2da4ac160930ebe75bb67015ff575a973ebbfdfe7b40bbc
-  data.tar.gz: a86b9615c22f44519bafb165fd4503b16c4cab05be70789bb3fdf3a8ade5a113
+  metadata.gz: eaed16618a5b214257cde6a34a750b61786c40adcddca7ebb8bb80b3e91014ce
+  data.tar.gz: c7593de893e79d23c042e03026902e04152552d0db79cbf63daba199d8567940
 SHA512:
-  metadata.gz: 5e05122ab0d685cb9f6e3bb3c793c822f23752620f11027912fd298d5302d8a019d9117124f8d8cd4b53e20afb1c3c0a0153f26d17f3131a631bf1ac16a67359
-  data.tar.gz: 731890e6307ff91e1a7a5d6d6c4b22085f6e28e1bad95de9591ec6fcb68a164a4b32d960a1fa9dc1d931788a47bbdc067038adc7b28d5f48add940ebca1e6de2
+  metadata.gz: 45db8c8062c76b474eef6d7d0c80554fc402ef320df9fbd4645d64f982ad178a6d853b79fafdda01d63b0399228cbf65dadef8948de53b2fdcefb8bff304209a
+  data.tar.gz: e17eca676c394e5900baa609641de46e6c8033b3b3ccf2c8d9257678982717082a5b51eb66672586bdf933c8c4e55a513bc4c49be46524d473373716a3ba4a5b

data/lib/fushin/models/website.rb

@@ -25,12 +25,12 @@ module Fushin
         "https://urlscan.io/domain/#{domain}"
       end
 
-      def url_for_vt
-        uri.path.empty? ? "#{url}/" : url
+      def normalized_url
+        @normalized_url ||= uri.path.empty? ? "#{url}/" : url
       end
 
       def vt_link
-        "https://www.virustotal.com/#/url/#{Digest::SHA256.hexdigest(url_for_vt)}"
+        "https://www.virustotal.com/#/url/#{Digest::SHA256.hexdigest(normalized_url)}"
       end
 
       def to_attachements

data/lib/fushin/monitor.rb

@@ -14,7 +14,6 @@ module Fushin
         attachements = [].tap do |out|
           out << item.post.btcs.map(&:to_attachements)
           out << item.post.urls.map(&:to_attachements)
-          out << item.post.links.map(&:to_attachements)
           out << item.post.attachements.map(&:to_attachements)
         end.flatten
         attachements << { text: "IoC is not found." } if attachements.empty?

data/lib/fushin/posts/post.rb

@@ -28,33 +28,26 @@ module Fushin
         end.first
       end
 
-      def main_text
-        @main_text ||= [].tap do |out|
-          detection = CharlockHolmes::EncodingDetector.detect(main.text)
-          out << CharlockHolmes::Converter.convert(main.text, detection[:encoding], "UTF-8")
-        end.first
-      end
-
       def btcs
-        @btcs ||= main_text.scan(/\b[13][a-km-zA-HJ-NP-Z0-9]{26,33}\b/).uniq.map do |address|
+        @btcs ||= main.text.scan(/\b[13][a-km-zA-HJ-NP-Z0-9]{26,33}\b/).uniq.map do |address|
           Models::BTC.new(address)
         end
       end
 
       def urls
-        @urls ||= main_text.scan(UrlRegex.get(scheme_required: true, mode: :parsing)).uniq.map do |url|
+        @urls ||= (urls_in_text + links).uniq.map do |url|
           next if whitelisted_domain?(url)
 
           Models::Website.new(url)
-        end.compact
+        end.compact.uniq(&:normalized_url)
       end
 
-      def links
-        @links ||= main.css("a").map { |a| a.get("href") }.compact.uniq.map do |url|
-          next if whitelisted_domain?(url)
+      def urls_in_text
+        @urls_in_text ||= main.text.scan(UrlRegex.get(scheme_required: true, mode: :parsing))
+      end
 
-          Models::Website.new(url)
-        end.compact
+      def links
+        @links ||= main.css("a").map { |a| a.get("href") }.compact
       end
 
       def attachements
@@ -90,7 +83,8 @@ module Fushin
         res = HTTP.get(url)
         return nil unless res.code == 200
 
-        res.body.to_s
+        detection = CharlockHolmes::EncodingDetector.detect(res.body.to_s)
+        CharlockHolmes::Converter.convert(res.body.to_s, detection[:encoding], "UTF-8")
       end
     end
   end

data/lib/fushin/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Fushin
-  VERSION = "0.3.1"
+  VERSION = "0.3.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fushin
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Manabu Niseki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-02-03 00:00:00.000000000 Z
+date: 2019-02-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler