ruboty-ymcrawl 0.0.8 → 0.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c8e694eea5a87e30d53596aa02cf75da8697d358
-  data.tar.gz: 49b70cb6cdf93492e8f9cc9c0a13221bc03754b2
+  metadata.gz: a30b00f8ef3a0f7058f46fdd58206e0170d95592
+  data.tar.gz: 9526594621419c35b4e9ed73dbdd1051ea1536a0
 SHA512:
-  metadata.gz: 3e19818fb8ae5ed3a5efb50530754cc58ac6d307753d2b78ca8f0092d6ba411ca98ef9b205eafda4671d04273da096e136bf0c721bb8cb440a08877ac074f596
-  data.tar.gz: 513f9cdce38e82e590232c0352c6145f32836eb02fb969eea15fd5b8f217c4651fbaf78a8dfc371458d6e8156f2f5f63892c00fd94df7dcf816d3eab617d3f7e
+  metadata.gz: 2a1f83c1f5b8572579b8fa86451a1d9d3387e3bfeaca7174fa4ef78bff2dce68375a433ce2737e9464f583242e59eadc43acc564f256c30e76b0355139ec9ca2
+  data.tar.gz: 4a2909657f41bb3a11e90962e81c5ba966c6ac139dc9401785851503bdd14baf59462b3a97bfae3646d1f76b129e29213c3f12268a0e5ddca9d77e7a0b906693

data/lib/ruboty/handlers/ymcrawl.rb

@@ -11,7 +11,7 @@ module Ruboty
       end
 
       def get_crawl
-        @crawl = YMCrawl::Core.new if @crawl == nil
+        @crawl = Ruboty::YMCrawl::Core.new if @crawl == nil
         @crawl
       end
     end
@@ -24,14 +24,6 @@ module Ruboty
       end
     end
 
-    class Test < Base
-      on(/test\z/i, name: "test", description: "Return test message")
-
-      def test(message)
-        message.reply("This is test!!")
-      end
-    end
-
     class Crawl < Base
       on(
         /crawl ?(?<url>.+)?\z/i,
@@ -47,6 +39,7 @@ module Ruboty
       end
 
       def crawl(message)
+        puts "crawl start in handlers"
         url = (message[:url] == nil) ? "-- please set url --" : message[:url]
         begin
           crawl = CrawlManager.instance.get_crawl

data/lib/ruboty/ymcrawl/main.rb

@@ -7,155 +7,161 @@ require 'find'
 require 'kconv'
 require 'json-schema'
 
-module YMCrawl
-	ORG_SETTING_FILE_PATH     = "YMCrawlfile"
-	SETTING_FILE_PATH         = "#{ORG_SETTING_FILE_PATH}"
-	SCHEMA_FILE_PATH          = "YMCrawl_schema.json"
-	UPLOADER_SCHEMA_FILE_PATH = "uploader_schema.json"
-	SITE_JSON_PATH            = "site.json"
-  
-  class DataManager
-
-    include Singleton
-
-    def initialize
-    	@setting = JSON.parse( File.open(SETTING_FILE_PATH).read)
-    	puts "YMCrawlfile valid"
-    	puts JSON::Validator.fully_validate(SCHEMA_FILE_PATH, @setting, :insert_defaults => true).to_s
-    	@sites = get_sites_json(SITE_JSON_PATH)
-    	File.write( SITE_JSON_PATH, JSON.unparse(@sites) ) unless FileTest.exist?(SITE_JSON_PATH)
-    	puts "uploader valid"
-    	puts JSON::Validator.fully_validate(UPLOADER_SCHEMA_FILE_PATH, get_uploader_data, :insert_defaults => true).to_s
-    end
-
-    # 各サイトごとの、画像取得のためのcssセレクタを記載したjsonをファイルから取得して返す
-    def get_sites_json(path)
-    	path = FileTest.exist?(path) ? path : @setting["site_json"]
-    	puts "reading site json file from #{path}"
-    	JSON.parse( open(path).read)
-    end
-
-    # URLのドメインに合致するsite情報を返す
-    def get_current_uploder_info(url)
-    	host = URI(url).host
-    	# ハッシュのkeyがs[0],valueがs[1]に入る
-    	@sites.each{ |s| return s[1] if s[1]["host"] == host }
-    	return @sites["default"]
-    end
-
-    def update_access_token(uploader_name, access_token)
-    	@setting["uploader"][uploader_name]["access_token"] = access_token if @setting["uploader"][uploader_name] != access_token
-    	puts "setting: #{@setting}"
-    	open(SETTING_FILE_PATH, 'w') do |io|
-    		JSON.dump(@setting, io)
-    	end
-    end
-
-    def get_setting;                  @setting                                         end
-    def get_save_to;                  @setting["save_to"]                              end
-    def get_uploader_data;            @setting["uploader"][get_save_to]                end
-    def get_current_access_token;     get_uploader_data["access_token"]                end
-    def get_current_app_key;          ENV["#{@setting["save_to"].upcase }_APP_KEY"]    end
-    def get_current_app_secret;       ENV["#{@setting["save_to"].upcase }_APP_SECRET"] end
-  end
-
-	class Core
-		def initialize
-			@data = DataManager.instance
-			if @data.get_save_to != "local"
-				@uploader = Uploader.new(@data.get_save_to, @data.get_current_app_key, @data.get_current_app_secret, @data.get_current_access_token)
+module Ruboty
+	module YMCrawl
+		ORG_SETTING_FILE_PATH     = "YMCrawlfile"
+		SETTING_FILE_PATH         = "#{ORG_SETTING_FILE_PATH}"
+		SCHEMA_FILE_PATH          = "YMCrawl_schema.json"
+		UPLOADER_SCHEMA_FILE_PATH = "uploader_schema.json"
+		SITE_JSON_PATH            = "site.json"
+	  
+	  class DataManager
+
+	    include Singleton
+
+	    def initialize
+	    	@setting = JSON.parse( File.open(SETTING_FILE_PATH).read)
+	    	puts "setting: #{@setting}"
+	    	puts "YMCrawlfile valid"
+	    	puts JSON::Validator.fully_validate(SCHEMA_FILE_PATH, @setting, :insert_defaults => true).to_s
+	    	@sites = get_sites_json(SITE_JSON_PATH)
+	    	File.write( SITE_JSON_PATH, JSON.unparse(@sites) ) unless FileTest.exist?(SITE_JSON_PATH)
+	    	puts "uploader valid"
+	    	puts "uploder data: #{get_uploader_data}"   
+	    	puts JSON::Validator.fully_validate(UPLOADER_SCHEMA_FILE_PATH, get_uploader_data, :insert_defaults => true).to_s
+	    end
+
+	    # 各サイトごとの、画像取得のためのcssセレクタを記載したjsonをファイルから取得して返す
+	    def get_sites_json(path)
+	    	path = FileTest.exist?(path) ? path : @setting["site_json"]
+	    	puts "reading site json file from #{path}"
+	    	JSON.parse( open(path).read)
+	    end
+
+	    # URLのドメインに合致するsite情報を返す
+	    def get_current_uploder_info(url)
+	    	host = URI(url).host
+	    	# ハッシュのkeyがs[0],valueがs[1]に入る
+	    	@sites.each{ |s| return s[1] if s[1]["host"] == host }
+	    	return @sites["default"]
+	    end
+
+	    def update_access_token(uploader_name, access_token)
+	    	@setting["uploader"][uploader_name]["access_token"] = access_token if @setting["uploader"][uploader_name] != access_token
+	    	puts "setting: #{@setting}"
+	    	open(SETTING_FILE_PATH, 'w') do |io|
+	    		JSON.dump(@setting, io)
+	    	end
+	    end
+
+	    def get_setting;                  @setting                                         end
+	    def get_save_to;                  @setting["save_to"]                              end
+	    def get_uploader_data;            @setting["uploader"][get_save_to]                end
+	    def get_current_access_token;     get_uploader_data["access_token"]                end
+	    def get_current_app_key;          ENV["#{@setting["save_to"].upcase }_APP_KEY"]    end
+	    def get_current_app_secret;       ENV["#{@setting["save_to"].upcase }_APP_SECRET"] end
+	  end
+
+		class Core
+			def initialize
+				@data = DataManager.instance
+				if @data.get_save_to != "local"
+					@uploader = Uploader.new(@data.get_save_to, @data.get_current_app_key, @data.get_current_app_secret, @data.get_current_access_token)
+				end
 			end
-		end
 
-		def start(urls); upload crawl(urls) end
+			def start(urls); upload crawl(urls) end
 
-		# 画像をクロールして保存する。保存したファイルのパスを返す。
-		def crawl(urls)
-			ncrawler  = Crawler.new(@data.get_setting["dst_dir"], @data.get_current_uploder_info(urls[0]), @data.get_setting["wait_time"])
-			urls.map{ |v| ncrawler.save_images(v) }
-		end
+			# 画像をクロールして保存する。保存したファイルのパスを返す。
+			def crawl(urls)
+				puts "in crawl"
+				ncrawler  = Crawler.new(@data.get_setting["dst_dir"], @data.get_current_uploder_info(urls[0]), @data.get_setting["wait_time"])
+				urls.map{ |v| ncrawler.save_images(v) }
+			end
 
-		# 画像を指定した先へアップロード
-		def upload(file_dirs)
-			setting  = @data.get_setting
-			return nil if @data.get_save_to == "local"
-
-			@uploader.login(@data.get_current_access_token)
-			zip_paths = file_dirs.map{ |dir| zip_dir(dir) }
-			encode = (ENV["LANG"] == nil) ? "utf-8" : ENV["LANG"]
-			begin
-				file_dirs.each{ |dir| FileUtils::remove_entry_secure( dir.force_encoding(encode) ) }
-			rescue
-				if encode != "ascii-8bit"
-					encode = "ascii-8bit"
-					retry
+			# 画像を指定した先へアップロード
+			def upload(file_dirs)
+				puts "in upload"
+				setting  = @data.get_setting
+				return nil if @data.get_save_to == "local"
+
+				@uploader.login(@data.get_current_access_token)
+				zip_paths = file_dirs.map{ |dir| zip_dir(dir) }
+				encode = (ENV["LANG"] == nil) ? "utf-8" : ENV["LANG"]
+				begin
+					file_dirs.each{ |dir| FileUtils::remove_entry_secure( dir.force_encoding(encode) ) }
+				rescue
+					if encode != "ascii-8bit"
+						encode = "ascii-8bit"
+						retry
+					end
 				end
+				share_paths = []
+				zip_paths.each do |path|
+					puts "uploading #{path} to dropbox"
+					put_result = @uploader.put([path])
+					File::delete(path)
+					share_paths << @uploader.get_share_link(put_result["path"])["url"]
+				end
+				return share_paths
 			end
-			share_paths = []
-			zip_paths.each do |path|
-				puts "uploading #{path} to dropbox"
-				put_result = @uploader.put([path])
-				File::delete(path)
-				share_paths << @uploader.get_share_link(put_result["path"])["url"]
-			end
-			return share_paths
-		end
 
-		# 指定されたディレクトリ以下のファイルをzipにする。返り値はzipのパス
-		def zip_dir(src)
-			dst = "#{src}.zip"
-			Zip::Archive.open(dst, Zip::CREATE) do |ar|
-				Dir.glob("#{src}/*").each do |item|
-					ar.add_file(item)
+			# 指定されたディレクトリ以下のファイルをzipにする。返り値はzipのパス
+			def zip_dir(src)
+				dst = "#{src}.zip"
+				Zip::Archive.open(dst, Zip::CREATE) do |ar|
+					Dir.glob("#{src}/*").each do |item|
+						ar.add_file(item)
+					end
 				end
+				dst
 			end
-			dst
-		end
 
-		def get_uploader; @uploader end
-	end
-
-	# ファイルをアップロードする先を抽象化したクラス
-	class Uploader
-		def initialize(name, app_key, app_secret, access_token = nil)
-			@name         = name
-			@app_key      = app_key
-			@app_secret   = app_secret
-			@access_token = access_token
-			@c_uploader   = create_uploader
+			def get_uploader; @uploader end
 		end
 
-		# 引数に応じてアップロード先のインスタンスを返す
-		def create_uploader
-			return @c_uploader unless @c_uploader == nil
-			if @name == "dropbox"
-				@c_uploader = DropboxManager.new(@app_key, @app_secret) 
-				return @c_uploader
+		# ファイルをアップロードする先を抽象化したクラス
+		class Uploader
+			def initialize(name, app_key, app_secret, access_token = nil)
+				@name         = name
+				@app_key      = app_key
+				@app_secret   = app_secret
+				@access_token = access_token
+				@c_uploader   = create_uploader
 			end
-			raise ArgumentError("uploader #{@name} is not found")
-		end
 
-		def access_token?; @access_token != "" and @access_token != nil end
+			# 引数に応じてアップロード先のインスタンスを返す
+			def create_uploader
+				return @c_uploader unless @c_uploader == nil
+				if @name == "dropbox"
+					@c_uploader = DropboxManager.new(@app_key, @app_secret) 
+					return @c_uploader
+				end
+				raise ArgumentError("uploader #{@name} is not found")
+			end
 
-		def verify_auth_code(auth_code)
-			@access_token = @c_uploader.get_access_token(auth_code)
-		end
+			def access_token?; @access_token != "" and @access_token != nil end
 
-		def login(token = nil)
-			@access_token = (token == nil) ? @access_token : token
-			puts "access token: #{@access_token}"
-			puts "---- access token isn't set when login!!!! ----" if token ==nil
-		  @c_uploader.login(token)
-		end
+			def verify_auth_code(auth_code)
+				@access_token = @c_uploader.get_access_token(auth_code)
+			end
 
-		def get_access_token_url
-			error = "---- YMCrawl publishing new access token url. But you already have access token. ----" 
-			puts error if @access_token != nil and @access_token != ""
-			@c_uploader.get_auth_code_url
-		end
+			def login(token = nil)
+				@access_token = (token == nil) ? @access_token : token
+				puts "access token: #{@access_token}"
+				puts "---- access token isn't set when login!!!! ----" if token ==nil
+			  @c_uploader.login(token)
+			end
 
-		def get_name;    @name                    end
-		def put(command) @c_uploader.put(command) end
-		def get_share_link(path) @c_uploader.get_share_link(path) end
+			def get_access_token_url
+				error = "---- YMCrawl publishing new access token url. But you already have access token. ----" 
+				puts error if @access_token != nil and @access_token != ""
+				@c_uploader.get_auth_code_url
+			end
+
+			def get_name;    @name                    end
+			def put(command) @c_uploader.put(command) end
+			def get_share_link(path) @c_uploader.get_share_link(path) end
+		end
 	end
-end
+end

data/lib/ruboty/ymcrawl/version.rb

@@ -1,5 +1,5 @@
 module Ruboty
   module Ymcrawl
-    VERSION = "0.0.8"
+    VERSION = "0.0.9"
   end
 end

data/lib/ruboty/ymcrawl.rb

@@ -1,4 +1,3 @@
-# require "ruboty"
 require "ruboty/handlers/ymcrawl"
 require "ruboty/ymcrawl/crawler"
 require "ruboty/ymcrawl/dropbox"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruboty-ymcrawl
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
 platform: ruby
 authors:
 - mpk
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-12 00:00:00.000000000 Z
+date: 2014-12-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable