ruboty-ymcrawl 0.0.1 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ff618b303eaf36d9fb1702df229d4bef5a97b76b
-  data.tar.gz: c2aaefedce57faf5432d349f15bcec566bc98a33
+  metadata.gz: f1fa7723d9cb543e8a2be4c47f7639f579798f33
+  data.tar.gz: 57d00ba352089fc46a5f337e525424b2eba68dc4
 SHA512:
-  metadata.gz: 2ea55a004dc5af6c3d30c3eedf5856e3a2e270fddee36e6abd24cfede67a2541189d167dc916b1234394c947aee340d963860944e642ccb8bfa1109dcb9be664
-  data.tar.gz: 27f054948dfdbfb4515b72242976b564f8104f29f785d488a0f8b622e26203563bac20b25f5e277a20f2c068cabe273e2050575f19b2ac018595c6b0c395c688
+  metadata.gz: 2a388ae8e594e16e25721f92a22dcf42d9845986b13d0880d1d9d7961edc69452a2ce88f0672ecf9a8122349cd0dd1acb7ad3088907c0702eda78863c24a16fb
+  data.tar.gz: c26de2ae35453342d9d9e63a83880df8ab4adb6d07e528728a845b2a60ee826dde6f2c69816f7e2652f8a6ac31621bf552d6c3a69fc6951f004776112c32d1fb

data/lib/ruboty/handlers/ymcrawl.rb

@@ -1,15 +1,88 @@
+require_relative 'src/main'
+require 'singleton'
+
 module Ruboty
   module Handlers
-    class YMCrawl < Base
+
+    class CrawlManager
+      include Singleton
+      def initialize
+        @crawl = nil
+      end
+
+      def get_crawl
+        @crawl = YMCrawl::Core.new if @crawl == nil
+        @crawl
+      end
+    end
+
+    class Hello < Base
+      on /hello\z/i, name: "hello", description: "Return hello"
+
+      def hello(message)
+        message.reply("hello!!")
+      end
+    end
+
+    class Crawl < Base
       on(
-        /crawl\z/,
-        name: 'crawl',
-        description: 'image crawler'
+        /crawl ?(?<url>.+)?\z/i,
+        name: "crawl",
+        description: "crawl image"
       )
 
-      def analyze(message)
-        Ruboty::YMCrawl::Actions::Analyze.new(message).call
+      def get_access_token_message(url)
+        return "You don't have access token.
+        1. Go to: #{url}
+        2. Click \"Allow\" (you might have to log in first).
+        3. reply to bot as \"@bot dropbox:auth (auth_code) \""
+      end
+
+      def crawl(message)
+        url = (message[:url] == nil) ? "-- please set url --" : message[:url]
+        begin
+          crawl = CrawlManager.instance.get_crawl
+          uploader = crawl.get_uploader
+
+          # upload先がlocal以外かつアクセストークンが取得されていない場合は、取得先URLを示して終了
+          if not uploader.access_token? and uploader.get_name != "local"
+            message.reply( get_access_token_message( uploader.get_access_token_url ) )
+            return nil
+          end
+
+          message.reply("rubot is crawling from #{url}")
+          zip_paths = crawl.start([url])
+          message.reply("get zip file => #{zip_paths}")
+        rescue URI::InvalidURIError => ex
+          puts ex
+          message.reply("URL is invalid. please retry.")
+        rescue => ex
+          puts "error raise in Crawl.crawl"
+          puts ex
+          message.reply("Sorry, error occurred.")
+          message.reply("Please feedback this error to niboshiporipori@gmail.com")
+          message.reply(ex)
+        end
       end
     end
+
+    class VerifyAuthCode < Base
+      on(
+        /dropbox:auth ?(?<auth_code>.+)?\z/i,
+        name: "verify_auth_code",
+        description: "add access token by auth code"
+      )
+
+      def verify_auth_code(message)
+        auth_code = (message[:auth_code] == nil) ? "-- please set auth_code --" : message[:auth_code]
+        uploader = CrawlManager.instance.get_crawl.get_uploader
+        access_token = uploader.verify_auth_code(auth_code)
+        YMCrawl::DataManager.instance.update_access_token(uploader.get_name, access_token)
+
+        message.reply("You added access token!")
+        message.reply("Try clawling again!")
+      end
+    end
+
   end
 end

data/lib/ruboty/ymcrawl/crawler.rb

@@ -0,0 +1,208 @@
+require 'open-uri'
+require 'nokogiri'
+require 'kconv'
+require 'addressable/uri'
+require 'singleton'
+
+module YMCrawl
+	# URLに関する処理をまとめたクラス
+	class URLUtil
+		def self.normalize_url(url)
+			puts "---- URL is null in normalize_url!!!!!!!!!!!!! ----" if url == nil
+			Addressable::URI.parse(url).normalize.to_s
+		end
+	end
+
+	# CSSセレクタを表すクラス
+	class Selector
+		def initialize(css)
+			@selector = css
+		end
+		
+		def to_s ;@selector end
+
+		# セレクタの一番最後のタグが何かを返す。擬似クラスなどは取り除く
+		def get_last_tag
+			# 一番最後の要素だけを返す。(擬似クラスなどは省く)
+			@selector.split(/\s|\+|>/).last.split(/:|,|\[|\.|#/).first
+		end
+	end
+
+	# ホストごとの処理を管理するクラス
+	class HostManager
+		include Singleton
+		DEFAULT_WAIT_TIME = 2
+		def initialize
+			@host_list = {}
+			@wait_time = DEFAULT_WAIT_TIME
+		end
+
+		def set_wait_time(wait_time) @wait_time = wait_time end
+
+		# 最後にアクセスした日時を取得する
+		def wait(url)
+			host = URI( URLUtil.normalize_url(url) ).host
+			unless @host_list[host] == nil then
+				time_diff = Time.now - @host_list[host]
+				puts "sleep: #{sleep(@wait_time - time_diff)}sec." if time_diff < @wait_time
+			end
+			@host_list[host] = Time.now
+		end
+	end
+
+	# あるURLから取得できるHTMLドキュメントを抽象化したクラス
+	class Page
+		class PageError < StandardError; end
+		def initialize(url)
+			@url = url
+			@doc = get_doc
+		end
+
+		# 指定したcssセレクタに合致する要素を表すクラスの配列を返す
+		def search_elements(selector) @doc.css(selector).map{ |doc| Element.new(doc) } end
+
+		private
+		# 与えられたURLをパースして返す
+		def get_doc
+			puts "get_doc from #{@url}"
+			HostManager.instance.wait(@url)
+			html = open(URLUtil.normalize_url(@url), "r:binary").read
+			Nokogiri::HTML(html.toutf8, nil, 'utf-8')
+		rescue OpenURI::HTTPError => ex
+			puts "failed URL: #{@url}"
+			puts "HTTP Error message: #{ex.message}"
+			raise PageError.new(ex.message)
+		end
+	end
+
+	# セレクタにより抽出されたPageの一部を表すクラス
+	class Element
+		def initialize(doc) @doc = doc end
+
+		def get_url; @doc["href"] end
+
+		# 画像へのURLを返す
+		def get_image_url
+			return @doc["href"] if @doc.name == "a"
+			return @doc["src"]  if @doc.name == "img"
+			raise ArgumentError, "in Element"
+		end
+
+		# 画像のタイトルを返す
+		def get_image_title
+			title = (@doc.name == "img") ? @doc["title"] : @doc.content
+			(title == nil) ? "noname" : title
+		end
+
+		# 記事タイトルを返す
+		def get_title; @doc.content end
+
+		# 記事が何ページまであるかを返す
+		def get_page_index_max; @doc.content.to_i end
+
+		# 対象に応じてURLを返す
+		def get_content(target)
+			return get_url            if target == :url
+			return get_image_url      if target == :image
+			return get_image_title    if target == :image_title
+			return get_title          if target == :title
+			return get_page_index_max if target == :page_index_max
+		end
+	end
+
+	# 画像のスクレイピングを行うクラス
+	class Crawler
+		INDEX_STR = "{index}" # jsonファイルでINDEX番号が入る場所を表す文字列
+
+		def initialize(dir, site_data, wait_time)
+			HostManager.instance.set_wait_time(wait_time)
+			@selectors = {}
+			@selectors[:image]          = site_data["css"]["image"].map          { |s| Selector.new(s) }
+			@selectors[:image_title]    = site_data["css"]["image_title"].map    { |s| Selector.new(s) }
+			@selectors[:title]          = site_data["css"]["title"].map          { |s| Selector.new(s) }
+			@selectors[:page_index_max] = site_data["css"]["page_index_max"].map { |s| Selector.new(s) }
+			@page_index_min             = site_data["page_index_min"]
+			@next_page_appendix         = (site_data["next_page_appendix"] == nil) ? "" : site_data["next_page_appendix"]
+			@dir = dir
+		end
+		
+		# 与えられたcssセレクタから画像を抽出する
+		def save_images(original_url)
+			dst_dir = "#{@dir}/#{get_contents(original_url, :title).first}"
+			(@page_index_min..get_page_index_max(original_url) ).each do |page_index|
+				url = "#{original_url}#{get_next_page_appendix_with_index(page_index)}"
+				get_contents(url, :image).zip(get_contents(url, :image_title)) do |url, title|
+					save_image(dst_dir, url, title) unless url == nil
+				end
+			end
+			dst_dir
+		end
+		
+		private
+		# ファイル名が既にimgディレクトリに存在していた場合はインデックスを付与する
+		def get_unique_name(dir, org_name)
+			basename = (org_name == nil) ? "noname" : File.basename(org_name, '.*')
+			ext = File.extname(org_name)
+			return "#{basename}#{ext}" unless FileTest.exist?("#{dir}/#{basename}#{ext}")
+			index = 1
+			retname = "#{basename}#{index}#{ext}"
+			while FileTest.exist?("#{dir}/#{retname}") do
+				index = index + 1
+				retname = "#{basename}#{index}#{ext}"
+			end
+			return retname
+		end
+
+		# 指定されたリンク先の画像を保存する
+		def save_image(dst_dir, url, title)
+			puts "src: #{url}"
+			# ready filepath
+			filename = "#{title}#{File.extname(url)}"
+			filePath = "#{dst_dir}/#{get_unique_name(dst_dir, filename)}"
+			HostManager.instance.wait(url)
+			# fileName folder if not exist
+			FileUtils.mkdir_p(dst_dir) unless FileTest.exist?(dst_dir)
+			# write image adata
+			begin
+				open(filePath, 'wb') do |output|
+					puts "dst: #{filePath}"
+					open(URLUtil.normalize_url(url)) do |data|
+						output.write(data.read)
+					end
+				end
+			rescue # ファイルが存在しないなどの理由で例外が発生した場合は、生成した画像を削除
+				puts "image not exist."
+				File.delete filePath
+			end
+		end
+
+		# URLに付加する文字列を返す
+		def get_next_page_appendix_with_index(index)
+			return "" if @next_page_appendix == ""
+			@next_page_appendix.gsub("{index}", index.to_s)
+		end
+
+		# 記事の最大ページを取得する
+		def get_page_index_max(url)
+			# page_index_maxのcssが空文字だとget_contentsがエラーになるので、最初にチェック
+			return @page_index_min if @next_page_appendix == ""
+			page_index_max = get_contents(url, :page_index_max)
+			return @page_index_min if page_index_max.length == 0
+			(page_index_max.first.kind_of?(Integer)) ? page_index_max.first : @page_index_min
+		end
+
+		# 与えられたURLから、セレクタに従って画像のURLを返す
+		def get_contents(url, target, nest = 0)
+			selector = @selectors[target][nest]
+			if nest >= (@selectors[target].length - 1)
+				return Page.new(url).search_elements(selector.to_s).map{ |cn| cn.get_content(target) } 
+			end
+			# 得られたURLそれぞれに対して次のセレクタを実行する
+			contents = Page.new(url).search_elements(selector.to_s).map{ |cn| cn.get_content(:url) } 
+			contents.map{ |c| get_contents(c, target, nest + 1) }.flatten
+		rescue Page::PageError => ex
+			puts "error in get_contents #{ex}"
+			return nil
+		end
+	end
+end

data/lib/ruboty/ymcrawl/dropbox.rb

@@ -0,0 +1,60 @@
+# Install this the SDK with "gem install dropbox-sdk"
+require 'dropbox_sdk'
+module YMCrawl
+	class DropboxManager
+
+		def initialize(app_key, app_sec)
+			@app_key = app_key
+			@app_sec = app_sec
+			@client = nil
+			@access_token = nil
+		end
+
+		def login(arg_access_token = nil)
+			if not @client.nil?
+				puts "already logged in!"
+				return @access_token
+			end
+
+			@access_token = arg_access_token
+			begin
+				@client = DropboxClient.new(@access_token)
+				puts "account info: #{@client.account_info()}"
+				return @access_token
+			rescue DropboxError => ex
+				puts "---- access token is invalid ----"
+				return nil
+			end
+		end
+		
+		def get_auth_code_url
+			puts "web_auth is nil!!!!" if @web_auth == nil
+			@web_auth = DropboxOAuth2FlowNoRedirect.new(@app_key, @app_sec)
+			authorize_url = @web_auth.start()
+		end
+
+		def get_access_token(auth_code)
+            @web_auth.finish(auth_code)[0]
+		end
+
+		def put(command)
+			fname = command[0]
+
+			#If the user didn't specifiy the file name, just use the name of the file on disk
+			if command[1]
+				new_name = command[1]
+			else
+				new_name = File.basename(fname)
+			end
+
+			if fname && !fname.empty? && File.exists?(fname) && (File.ftype(fname) == 'file') && File.stat(fname).readable?
+				#This is where we call the the Dropbox Client
+				pp @client.put_file(new_name, open(fname))
+			else
+				puts "couldn't find the file #{ fname }"
+			end
+		end
+
+		def get_share_link(path) @client.shares(path) end
+	end
+end

data/lib/ruboty/ymcrawl/main.rb

@@ -0,0 +1,161 @@
+require_relative 'crawler'
+require_relative 'dropbox.rb'
+require 'optparse'
+require 'json'
+require 'zipruby'
+require 'find'
+require 'kconv'
+require 'json-schema'
+
+module YMCrawl
+	ORG_SETTING_FILE_PATH     = "YMCrawlfile"
+	SETTING_FILE_PATH         = "#{ORG_SETTING_FILE_PATH}"
+	SCHEMA_FILE_PATH          = "YMCrawl_schema.json"
+	UPLOADER_SCHEMA_FILE_PATH = "uploader_schema.json"
+	SITE_JSON_PATH            = "site.json"
+  
+  class DataManager
+
+    include Singleton
+
+    def initialize
+    	@setting = JSON.parse( File.open(SETTING_FILE_PATH).read)
+    	puts "YMCrawlfile valid"
+    	puts JSON::Validator.fully_validate(SCHEMA_FILE_PATH, @setting, :insert_defaults => true).to_s
+    	@sites = get_sites_json(SITE_JSON_PATH)
+    	File.write( SITE_JSON_PATH, JSON.unparse(@sites) ) unless FileTest.exist?(SITE_JSON_PATH)
+    	puts "uploader valid"
+    	puts JSON::Validator.fully_validate(UPLOADER_SCHEMA_FILE_PATH, get_uploader_data, :insert_defaults => true).to_s
+    end
+
+    # 各サイトごとの、画像取得のためのcssセレクタを記載したjsonをファイルから取得して返す
+    def get_sites_json(path)
+    	path = FileTest.exist?(path) ? path : @setting["site_json"]
+    	puts "reading site json file from #{path}"
+    	JSON.parse( open(path).read)
+    end
+
+    # URLのドメインに合致するsite情報を返す
+    def get_current_uploder_info(url)
+    	host = URI(url).host
+    	# ハッシュのkeyがs[0],valueがs[1]に入る
+    	@sites.each{ |s| return s[1] if s[1]["host"] == host }
+    	return @sites["default"]
+    end
+
+    def update_access_token(uploader_name, access_token)
+    	@setting["uploader"][uploader_name]["access_token"] = access_token if @setting["uploader"][uploader_name] != access_token
+    	puts "setting: #{@setting}"
+    	open(SETTING_FILE_PATH, 'w') do |io|
+    		JSON.dump(@setting, io)
+    	end
+    end
+
+    def get_setting;                  @setting                                         end
+    def get_save_to;                  @setting["save_to"]                              end
+    def get_uploader_data;            @setting["uploader"][get_save_to]                end
+    def get_current_access_token;     get_uploader_data["access_token"]                end
+    def get_current_app_key;          ENV["#{@setting["save_to"].upcase }_APP_KEY"]    end
+    def get_current_app_secret;       ENV["#{@setting["save_to"].upcase }_APP_SECRET"] end
+  end
+
+	class Core
+		def initialize
+			@data = DataManager.instance
+			if @data.get_save_to != "local"
+				@uploader = Uploader.new(@data.get_save_to, @data.get_current_app_key, @data.get_current_app_secret, @data.get_current_access_token)
+			end
+		end
+
+		def start(urls); upload crawl(urls) end
+
+		# 画像をクロールして保存する。保存したファイルのパスを返す。
+		def crawl(urls)
+			ncrawler  = Crawler.new(@data.get_setting["dst_dir"], @data.get_current_uploder_info(urls[0]), @data.get_setting["wait_time"])
+			urls.map{ |v| ncrawler.save_images(v) }
+		end
+
+		# 画像を指定した先へアップロード
+		def upload(file_dirs)
+			setting  = @data.get_setting
+			return nil if @data.get_save_to == "local"
+
+			@uploader.login(@data.get_current_access_token)
+			zip_paths = file_dirs.map{ |dir| zip_dir(dir) }
+			encode = (ENV["LANG"] == nil) ? "utf-8" : ENV["LANG"]
+			begin
+				file_dirs.each{ |dir| FileUtils::remove_entry_secure( dir.force_encoding(encode) ) }
+			rescue
+				if encode != "ascii-8bit"
+					encode = "ascii-8bit"
+					retry
+				end
+			end
+			share_paths = []
+			zip_paths.each do |path|
+				puts "uploading #{path} to dropbox"
+				put_result = @uploader.put([path])
+				File::delete(path)
+				share_paths << @uploader.get_share_link(put_result["path"])["url"]
+			end
+			return share_paths
+		end
+
+		# 指定されたディレクトリ以下のファイルをzipにする。返り値はzipのパス
+		def zip_dir(src)
+			dst = "#{src}.zip"
+			Zip::Archive.open(dst, Zip::CREATE) do |ar|
+				Dir.glob("#{src}/*").each do |item|
+					ar.add_file(item)
+				end
+			end
+			dst
+		end
+
+		def get_uploader; @uploader end
+	end
+
+	# ファイルをアップロードする先を抽象化したクラス
+	class Uploader
+		def initialize(name, app_key, app_secret, access_token = nil)
+			@name         = name
+			@app_key      = app_key
+			@app_secret   = app_secret
+			@access_token = access_token
+			@c_uploader   = create_uploader
+		end
+
+		# 引数に応じてアップロード先のインスタンスを返す
+		def create_uploader
+			return @c_uploader unless @c_uploader == nil
+			if @name == "dropbox"
+				@c_uploader = DropboxManager.new(@app_key, @app_secret) 
+				return @c_uploader
+			end
+			raise ArgumentError("uploader #{@name} is not found")
+		end
+
+		def access_token?; @access_token != "" and @access_token != nil end
+
+		def verify_auth_code(auth_code)
+			@access_token = @c_uploader.get_access_token(auth_code)
+		end
+
+		def login(token = nil)
+			@access_token = (token == nil) ? @access_token : token
+			puts "access token: #{@access_token}"
+			puts "---- access token isn't set when login!!!! ----" if token ==nil
+		  @c_uploader.login(token)
+		end
+
+		def get_access_token_url
+			error = "---- YMCrawl publishing new access token url. But you already have access token. ----" 
+			puts error if @access_token != nil and @access_token != ""
+			@c_uploader.get_auth_code_url
+		end
+
+		def get_name;    @name                    end
+		def put(command) @c_uploader.put(command) end
+		def get_share_link(path) @c_uploader.get_share_link(path) end
+	end
+end

data/lib/ruboty/ymcrawl/version.rb

@@ -1,5 +1,5 @@
 module Ruboty
   module Ymcrawl
-    VERSION = "0.0.1"
+    VERSION = "0.0.3"
   end
 end

data/lib/ruboty/ymcrawl.rb

@@ -1,9 +1,6 @@
+require "ruboty/handlers/ymcrawl"
+require "ruboty/ymcrawl/crawler"
+require "ruboty/ymcrawl/dropbox"
+require "ruboty/ymcrawl/main"
 require "ruboty/ymcrawl/version"
 
-module Ruboty
-  module Ymcrawl
-  	def self.hoge
-  	  puts "hello from ymcrawl hoge"
-  	end
-  end
-end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruboty-ymcrawl
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.3
 platform: ruby
 authors:
 - mpk
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-09 00:00:00.000000000 Z
+date: 2014-12-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -159,6 +159,9 @@ files:
 - Rakefile
 - lib/ruboty/handlers/ymcrawl.rb
 - lib/ruboty/ymcrawl.rb
+- lib/ruboty/ymcrawl/crawler.rb
+- lib/ruboty/ymcrawl/dropbox.rb
+- lib/ruboty/ymcrawl/main.rb
 - lib/ruboty/ymcrawl/version.rb
 - ruboty-ymcrawl.gemspec
 - spec/ruboty/ymcrawl_spec.rb