my-lib 0.0.1.1 → 0.0.1.3

data/Gemfile

@@ -13,4 +13,6 @@ group :development do
   gem "bundler", "> 1.0.0"
   gem "jeweler"
   gem "rcov", ">= 0"
+  gem "guard"
+  gem "guard-spork"
 end

data/Gemfile.lock

@@ -4,6 +4,11 @@ GEM
     diff-lcs (1.1.3)
     eventmachine (0.12.10)
     git (1.2.5)
+    guard (0.6.3)
+      thor (~> 0.14.6)
+    guard-spork (0.2.1)
+      guard (>= 0.2.2)
+      spork (>= 0.8.4)
     jeweler (1.6.4)
       bundler (~> 1.0)
       git (>= 1.2.5)
@@ -26,7 +31,9 @@ GEM
     rspec-expectations (2.6.0)
       diff-lcs (~> 1.1.2)
     rspec-mocks (2.6.0)
+    spork (0.8.5)
     term-ansicolor (1.0.6)
+    thor (0.14.6)
     webrobots (0.0.11)
       nokogiri (>= 1.4.4)
     yard (0.7.2)
@@ -37,6 +44,8 @@ PLATFORMS
 DEPENDENCIES
   bundler (> 1.0.0)
   eventmachine
+  guard
+  guard-spork
   jeweler
   mechanize
   rcov

data/TAGS

@@ -0,0 +1,66 @@
+
+(null),18
+def hogehoge
2,6
+
+lib/dev.rb,1335
+class MyObjectMyObject
24,488
+  def self.myrequiremyrequire
25,503
+class MyConfigMyConfig
33,665
+  def self.getget
35,718
+  def self.readread
42,795
+module RunPerSecModuleRunPerSecModule
63,1245
+  def run(sec)run
71,1556
+  def init_run_per_sec_moduleinit_run_per_sec_module
86,1798
+  def stop_runstop_run
91,1885
+  def before_run_loopbefore_run_loop
96,2014
+  def loop_hook_preloop_hook_pre
100,2103
+  def main_loopmain_loop
104,2181
+  def loop_hook_postloop_hook_post
108,2269
+  def after_run_loopafter_run_loop
112,2373
+module MyDBModuleMyDBModule
139,2874
+  def set_my_dbset_my_db
147,2981
+  def insert_DB(key='test_app',value='')insert_DB
168,3575
+module MyPusherModuleMyPusherModule
195,4206
+  def set_my_pusherset_my_pusher
201,4330
+  def push_pusher(app_name='test_app',data='test')push_pusher
216,4701
+module MyGCalModuleMyGCalModule
246,5321
+  def gcal_readgcal_read
249,5403
+  def gcal_write(eventdata)gcal_write
256,5509
+  def gcal_parse_2_jobsgcal_parse_2_jobs
269,5858
+  def gcal_checkout(event)gcal_checkout
290,6403
+  def serviceservice
297,6540
+module MyAtModuleMyAtModule
323,7102
+  def initializeinitialize
324,7120
+  def gcaljobs_2_atgcaljobs_2_at
328,7180
+  def jobs2at(job)jobs2at
333,7266
+  def _at_scriptpath(job)_at_scriptpath
346,7600
+  def _at_command(job)_at_command
350,7716
+
+lib/my-lib.rb,924
+class StringString
6,89
+class MyLoggerMyLogger
16,267
+  def self.ln(message)ln
22,367
+  def self.lw(message)lw
30,540
+  def self.beforebefore
38,717
+  def self.afterafter
44,843
+class MyJobAnisokuMyJobAnisoku
66,1426
+  def initialize(args = { })initialize
70,1471
+  def tokkakaritokkakari
84,1872
+  def secondsecond
103,2321
+  def thirdthird
124,2884
+  def kobetukobetu
143,3352
+  def videovideo
240,5755
+  def runrun
302,7444
+module MyMachineMyMachine
325,7804
+  def initializeinitialize
331,7871
+  def setupsetup
336,7941
+  def gogo
341,7991
+  def retry(job)retry
345,8048
+  def setupjobssetupjobs
351,8103
+  def setupmachinesetupmachine
355,8177
+class MyMachineAnisokuMyMachineAnisoku
369,8438
+  def initialize(args={ })initialize
381,8764
+  def gogo
397,9115
+  def finalize_filesfinalize_files
414,9443
+  def setupjobssetupjobs
428,9813
+  def should_stop_machine?should_stop_machine?
436,9948

data/VERSION

@@ -1 +1 @@
-0.0.1.1
+0.0.1.3

data/lib/1.rb

@@ -0,0 +1,4 @@
+#test
+def hoge
+  p :hoge
+end

data/lib/lib/anisoku.rb

@@ -0,0 +1,292 @@
+#-*- coding:utf-8 -*-
+
+#
+# Job Class for Fetch Anisoku
+# Function:
+#   access "youtubeanisoku1.blog106.fc2.com" and crawl web site.
+#   find a link to "say-move.org" and access "say-move.org".
+#   finary find video link of Animetion,then fetch video file to your local.
+#   save video directory is supplied by machine.
+# Notice:
+#   job is automatically generated on after another.
+#   This Class Needs to be handle by Machine Class
+# @example
+#   # inside machine class     
+#     job = MyJobAnisoku.new(
+#      :machine => self
+#     )
+#     job.run
+#
+
+class MyJobAnisoku
+
+  Version = "0.0.2"    
+
+  def initialize(args = { })
+    require 'rubygems'
+    require 'kconv'
+    require 'mechanize'
+    require 'net/http'
+    @a = args
+    @a[:url] ||= 'http://youtubeanisoku1.blog106.fc2.com/'
+    @a[:url] = URI.parse @a[:url] unless @a[:url].class == URI::HTTP
+    @agent = Mechanize.new
+    @a[:status] ||= :new
+    raise "job have no machine error"  unless @a[:machine]
+  end
+
+  # check kousin page
+  def tokkakari
+    print "Tokkakari".yellow
+    @agent.get @a[:url]
+    links_kousins = @agent.page.links_with(:text => /#{"更新状況".toutf8}/)
+    targs = []
+    links_kousins.each do |link|
+      targs << link.uri
+    end
+    targs.each_with_index do |link,i|
+      break if i > 6
+      job = MyJobAnisoku.new(
+        :url => link,
+        :status => :second,
+        :machine => @a[:machine]
+        )
+      @a[:machine].retry job
+    end
+    
+  end
+
+  # check shoukai page
+  def second
+    print "Second".yellow
+    @agent.get @a[:url]
+    links_kousin =  @agent.page/"/html/body/table/tr[2]/td/table/tr/td[2]/div[4]/ul/li/a/@href"
+    # links_kobetu 
+    links_kobetu = []
+    links_kousin.each do |link|
+      links_kobetu << $1  if link.value =~ /(http:\/\/youtubeanisoku.*)/
+    end
+
+    # make job for each links_kobetu
+    links_kobetu.each do |link|
+      job = MyJobAnisoku.new(
+        :url => link,
+        :status => :kobetu,
+        :machine => @a[:machine]
+        )
+      @a[:machine].retry job
+    end
+  end
+
+  #access say-move and make video job
+  def kobetu
+    print "Kobetu".yellow
+    @agent.get @a[:url]
+    _tt = @agent.page.title.gsub(' ★ You Tube アニ速 ★','')
+    limit = 5
+    urls = []
+    # acume url
+    nodeset_vs = @agent.page/"/html/body/table/tr[2]/td/table/tr/td[2]/div[4]/div[2]/a/@href"
+    _dd = []
+
+    nodeset_vs.each do |va|
+      _dd << $1  if va.value =~ /(http:\/\/say-move\.org\/comeplay\.php.*)/
+    end
+    _dd.reverse!
+
+    #hard coding for adjust fetch limit
+    _dd.each_with_index do |url,i|
+      break if i > limit
+      urls << url
+    end
+    
+    urls.each_with_index do |url,i|
+      job = MyJobAnisoku.new(
+        :url => url,
+        :title => _tt,
+        :status => :third,
+        :machine => @a[:machine]
+        )
+      @a[:machine].retry job
+    end
+  end
+
+  #access say-move and make video job
+  def third
+    print "Third".yellow
+    #sm has title and url
+
+    sm = { :title => @a[:title],:url => @a[:url]}
+    # debug fc2 video sm[:url] = "http://say-move.org/comeplay.php?comeid=217953"
+    @agent.get(sm[:url])
+    sm[:title] += @agent.page.title.gsub!('FC2 SayMove!','') 
+    set =  @agent.page/"/html/body/div/div[2]/div[7]/div[2]/input/@value"
+    if !set.empty?
+      sm[:videourl] = set[0].value 
+    else
+      set =  @agent.page/"/html/body/div/div[2]/div[3]/object/param[5]/@value"
+      fc2 = set[0].value.split('&')[1].split('=')[1]
+      unless fc2.nil?
+        job = MyJobAnisoku.new(
+          :url => sm[:url],
+          :fc2 => fc2,
+          :title => sm[:title],
+          :status => :fc2,
+          :machine => @a[:machine]
+          )
+        @a[:machine].retry job
+        return
+      else
+      end
+    end
+    
+    job = MyJobAnisoku.new(
+      :url => sm[:videourl],
+      :title => sm[:title],
+      :status => :video,
+      :machine => @a[:machine]
+      )
+     @a[:machine].retry job
+  end
+
+  def fc2
+    print "fc2".yellow
+    require 'digest'
+    # make md5 with magicword '_gGddgPfeaf_gzyr'
+    url = "http://video.fc2.com/ginfo.php?mimi=#{Digest::MD5.hexdigest(@a[:fc2] + '_gGddgPfeaf_gzyr')}&v=#{@a[:fc2]}&upid=#{@a[:fc2]}&otag=1"
+    url = `curl -# -L -R "#{url}"`
+    url =  url.split('&')[0].split('=')[1] + '?' + url.split('&')[1]
+    puts url.red.bold
+    job = MyJobAnisoku.new(
+      :url => url,
+      :title => @a[:title],
+      :status => :video,
+      :machine => @a[:machine]
+      )
+     @a[:machine].retry job
+  end
+
+  #fetch video
+  def video
+    print "video".yellow
+    # save video directory is supplied by machine.
+    savedir = @a[:machine].savedir
+    Dir.chdir savedir
+    filename = "#{@a[:title]}.mp4"
+    savepath = "#{savedir}/#{filename}"
+    # check fetch candidate had been already saved?
+    if File.exist?(savepath) && File.size(savepath) > 1024 * 1024 * 3
+      puts "File Already Saved ".yellow.bold  + savepath
+      return
+    else
+      puts "Fetching ".green.bold + savepath
+      MyLogger.ln "Fetch Attempt Start ".green.bold  + savepath
+    end
+    
+    # use curl command
+    # no need UA...
+    #    command = "curl -# -L -R -o '#{filename}' 'http://#{@a[:url].host}#{@a[:url].path}?#{@a[:url].query}' --user-agent 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:9.0a1) Gecko/20110901 Firefox/9.0a1'"
+    command = "curl -# -L -R -o '#{filename}' 'http://#{@a[:url].host}#{@a[:url].path}?#{@a[:url].query}' "
+
+    p command
+    system command
+  end
+
+  # run in thread
+  def run
+    t = Thread.new do
+      case @a[:status]
+      when :new then
+        tokkakari
+      when :second then
+        second
+      when :kobetu then
+        kobetu
+      when :third then
+        third
+      when :fc2 then
+        fc2
+      when :video then
+        video
+      end
+    end
+  end
+  
+end
+
+
+
+# 
+# Class of Machine by EventMachine.
+# this class controll jobs for Anisoku
+# @example
+#   machine = MyMachineAnisoku.new("YourVideoSaveDir")
+#   machine.setup
+#   machine.go
+#
+class MyMachineAnisoku
+  include MyMachine
+
+  Version = "0.0.1"    
+
+  # directory of save video files default "#{ENV['HOME']}/Desktop/video"
+  attr_accessor :savedir
+  
+  # set video save dir
+  # @param [Hash] args
+  # @option args [String] :savedir save dir
+  #                        default "#{ENV['HOME']}/Desktop/video"
+  def initialize(args={ })
+    super()
+    args[:savedir] ||= "#{ENV['HOME']}/Desktop/video2"
+    @savedir = args[:savedir]
+    begin
+      Dir::mkdir(@savedir, 0777)
+    rescue => ex
+      warn ex
+    end
+    require 'rubygems'
+    require 'eventmachine'
+    # @gaman controll eventmachine end
+    @gaman = 0
+  end
+
+  # machine go to run eventmachine
+  def go
+    EM.run do
+      EM.add_periodic_timer(0.00001) do
+#        print "loop".green
+        if should_stop_machine?        
+          finalize_files
+          EM.stop
+        end
+        @queue.pop.run unless @queue.empty?
+      end
+    end
+    puts "End of fetch".green.bold
+  end
+
+
+  # delete tiny fail files 
+  def finalize_files
+    command = "find #{@savedir} -size -1000k -type f -print0| xargs -0 rm -v "
+    exec(command)
+  end
+
+  private
+
+  # setup jobs
+  def setupjobs
+    ajob = MyJobAnisoku.new(
+      :machine => self
+      )
+    @queue.push ajob
+  end
+
+  # should stop machine or not
+  def should_stop_machine?
+    @gaman += 1 if @queue.empty?
+    print @gaman
+    return @queue.empty? && @gaman > 1500
+  end
+end

data/lib/lib/job.rb

@@ -0,0 +1,212 @@
+#-*- coding:utf-8 -*-
+# Job Class for Fetch Anisoku
+# Function:
+#   access "youtubeanisoku1.blog106.fc2.com" and crawl web site.
+#   find a link to "say-move.org" and access "say-move.org".
+#   finary find video link of Animetion,then fetch video file to your local.
+#   save video directory is supplied by machine.
+# Notice:
+#   job is automatically generated on after another.
+#   This Class Needs to be handle by Machine Class
+# @example
+#   # inside machine class     
+#     job = MyJobAnisoku.new(
+#      :machine => self
+#     )
+#     job.run
+#   
+class MyJobAnisoku
+
+  Version = "0.0.2"    
+
+  def initialize(args = { })
+    require 'rubygems'
+    require 'kconv'
+    require 'mechanize'
+    require 'net/http'
+    @a = args
+    @a[:url] ||= 'http://youtubeanisoku1.blog106.fc2.com/'
+    @a[:url] = URI.parse @a[:url] unless @a[:url].class == URI::HTTP
+    @agent = Mechanize.new
+    @a[:status] ||= :new
+    raise "job have no machine error"  unless @a[:machine]
+  end
+
+  # check kousin page
+  def tokkakari
+    print "Tokkakari".yellow
+    @agent.get @a[:url]
+    links_kousins = @agent.page.links_with(:text => /#{"更新状況".toutf8}/)
+    targs = []
+    links_kousins.each do |link|
+      targs << link.uri
+    end
+    targs.each_with_index do |link,i|
+      break if i > 6
+      job = MyJobAnisoku.new(
+        :url => link,
+        :status => :second,
+        :machine => @a[:machine]
+        )
+      @a[:machine].retry job
+    end
+    
+  end
+
+  # check shoukai page
+  def second
+    print "Second".yellow
+    @agent.get @a[:url]
+    links_kousin =  @agent.page/"/html/body/table/tr[2]/td/table/tr/td[2]/div[4]/ul/li/a/@href"
+    # links_kobetu 
+    links_kobetu = []
+    links_kousin.each do |link|
+      links_kobetu << $1  if link.value =~ /(http:\/\/youtubeanisoku.*)/
+    end
+
+    # make job for each links_kobetu
+    links_kobetu.each do |link|
+      job = MyJobAnisoku.new(
+        :url => link,
+        :status => :kobetu,
+        :machine => @a[:machine]
+        )
+      @a[:machine].retry job
+    end
+  end
+
+  #access say-move and make video job
+  def kobetu
+    print "Kobetu".yellow
+    @agent.get @a[:url]
+    _tt = @agent.page.title.gsub(' ★ You Tube アニ速 ★','')
+    limit = 5
+    urls = []
+    # acume url
+    nodeset_vs = @agent.page/"/html/body/table/tr[2]/td/table/tr/td[2]/div[4]/div[2]/a/@href"
+    _dd = []
+
+    nodeset_vs.each do |va|
+      _dd << $1  if va.value =~ /(http:\/\/say-move\.org\/comeplay\.php.*)/
+    end
+    _dd.reverse!
+
+    #hard coding for adjust fetch limit
+    _dd.each_with_index do |url,i|
+      break if i > limit
+      urls << url
+    end
+    
+    urls.each_with_index do |url,i|
+      job = MyJobAnisoku.new(
+        :url => url,
+        :title => _tt,
+        :status => :third,
+        :machine => @a[:machine]
+        )
+      @a[:machine].retry job
+    end
+  end
+
+  #access say-move and make video job
+  def third
+    print "Third".yellow
+    #sm has title and url
+
+    sm = { :title => @a[:title],:url => @a[:url]}
+    # debug fc2 video sm[:url] = "http://say-move.org/comeplay.php?comeid=217953"
+    @agent.get(sm[:url])
+    sm[:title] += @agent.page.title.gsub!('FC2 SayMove!','') 
+    set =  @agent.page/"/html/body/div/div[2]/div[7]/div[2]/input/@value"
+    if !set.empty?
+      sm[:videourl] = set[0].value 
+    else
+      set =  @agent.page/"/html/body/div/div[2]/div[3]/object/param[5]/@value"
+      fc2 = set[0].value.split('&')[1].split('=')[1]
+      unless fc2.nil?
+        job = MyJobAnisoku.new(
+          :url => sm[:url],
+          :fc2 => fc2,
+          :title => sm[:title],
+          :status => :fc2,
+          :machine => @a[:machine]
+          )
+        @a[:machine].retry job
+        return
+      else
+      end
+    end
+    
+    job = MyJobAnisoku.new(
+      :url => sm[:videourl],
+      :title => sm[:title],
+      :status => :video,
+      :machine => @a[:machine]
+      )
+     @a[:machine].retry job
+  end
+
+  def fc2
+    print "fc2".yellow
+    require 'digest'
+    # make md5 with magicword '_gGddgPfeaf_gzyr'
+    url = "http://video.fc2.com/ginfo.php?mimi=#{Digest::MD5.hexdigest(@a[:fc2] + '_gGddgPfeaf_gzyr')}&v=#{@a[:fc2]}&upid=#{@a[:fc2]}&otag=1"
+    url = `curl -# -L -R "#{url}"`
+    url =  url.split('&')[0].split('=')[1] + '?' + url.split('&')[1]
+    puts url.red.bold
+    job = MyJobAnisoku.new(
+      :url => url,
+      :title => @a[:title],
+      :status => :video,
+      :machine => @a[:machine]
+      )
+     @a[:machine].retry job
+  end
+
+  #fetch video
+  def video
+    print "video".yellow
+    # save video directory is supplied by machine.
+    savedir = @a[:machine].savedir
+    Dir.chdir savedir
+    filename = "#{@a[:title]}.mp4"
+    savepath = "#{savedir}/#{filename}"
+    # check fetch candidate had been already saved?
+    if File.exist?(savepath) && File.size(savepath) > 1024 * 1024 * 3
+      puts "File Already Saved ".yellow.bold  + savepath
+      return
+    else
+      puts "Fetching ".green.bold + savepath
+      MyLogger.ln "Fetch Attempt Start ".green.bold  + savepath
+    end
+    
+    # use curl command
+    # no need UA...
+    #    command = "curl -# -L -R -o '#{filename}' 'http://#{@a[:url].host}#{@a[:url].path}?#{@a[:url].query}' --user-agent 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:9.0a1) Gecko/20110901 Firefox/9.0a1'"
+    command = "curl -# -L -R -o '#{filename}' 'http://#{@a[:url].host}#{@a[:url].path}?#{@a[:url].query}' "
+
+    p command
+    system command
+  end
+
+  # run in thread
+  def run
+    t = Thread.new do
+      case @a[:status]
+      when :new then
+        tokkakari
+      when :second then
+        second
+      when :kobetu then
+        kobetu
+      when :third then
+        third
+      when :fc2 then
+        fc2
+      when :video then
+        video
+      end
+    end
+  end
+  
+end