autoweb 0.0.2

data/README.md

@@ -0,0 +1,91 @@
+# Welcome to Autoweb
+
+1. [Autoweb][homepage]能让更好地分析HTML中的数据
+ 	 你可以先使用Autoweb提供的DSL,针对HTML建模, 然后抓取数据
+2. 它能把你写的抓取程序分享出来, 他人可以通过命令行或者web界面使用
+
+### 集成进来的工具
+1. baidu mp3 下载器
+ 
+使用举例:
+
+下载 齐秦的大约在冬季, 在命令行输入:
+
+	autoweb baidump3 "大约在冬季 齐秦" ~/Download/mp3
+
+autoweb会自动搜索歌曲, 然后下载到指定的目录
+
+
+## Install Autoweb
+
+安装前需要安装以下工具:
+
+  * curl
+  * wget
+  * hpricot
+
+然后安装autoweb
+
+  gem install autoweb
+
+
+## Contributing
+
+### 页面建模 
+用到了css3作为页面元素定位语法, 参照: [css3语法介绍][w3c-css3-selector]
+
+	Page.define "BaiduMp3" do |page|
+    
+    # 搜索url模板
+		page.url_tpl = 'http://mp3.baidu.com/m?f=3&rf=idx&tn=baidump3&ct=134217728&lf=&rn=&word=#{word}&lm=-1&oq=go&rsp=1'
+		
+		page.subs "result", "#Tbs tr" do |sub| # 定义名为"result"的"sub page"
+
+			sub.ele "music", "td:nth(1) a"  #音乐链接
+			sub.ele "artist", "td:nth(2) a" #演唱者
+			sub.ele "album", "td:nth(3) a"  #专辑
+			sub.ele "lyrics", "td:nth(5) a" #歌词
+			sub.ele "size", "td:nth(7)"     #文件大小
+			sub.ele "format", "td:nth(8)"   #文件格式
+		end
+	end
+
+使用页面对象:
+ 
+	page = Page.pages["BaiduMp3"].parse(:word=>"大约在冬季")
+	first_mp3 = page["result"][1]
+	link = first_mp3["music"]
+	puts link[:href]
+
+更多实际代码, 参考 [commands/baidump3.rb][baidump3-codeb] 
+
+### 新建一个命令(和baidump3类似)
+
+将以下代码放到autoweb/commands/helloworld.rb下
+
+	module Autoweb::Command
+		class HelloWorld < Base  
+			def index
+				display "hello world!"
+			end
+		end
+	end
+
+直接运行 autoweb helloworld 即可
+
+更多实际代码, 参考 [commands/help.rb][help-code] 和 [commands/baidump3.rb][baidump3-code] 
+
+### 将代码提交到[autoweb][homepage]
+
+   请直接fork github上的autoweb, 提交ticket以及push request即可
+
+## License
+
+Autoweb released under the MIT license.
+
+[homepage]:http://dazuiba.github.com/autoweb
+[w3c-css3-selector]:http://wiki.github.com/mxcl/homebrew/installation
+[baidump3-code]:http://github.com/dazuiba/autoweb/blob/master/commands/baidump3.rb
+[help-code]:http://github.com/dazuiba/autoweb/blob/master/commands/help.rb
+
+

data/bin/autoweb

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift(File.dirname(__FILE__) + '/../lib')
+
+def display(msg)
+  STDOUT.print msg
+  STDOUT.flush
+end
+
+display "loading"
+require 'rubygems'
+display "..."
+require 'autoweb/command'
+display "...\n"
+
+
+at_exit{display "\n"}
+
+args = ARGV.dup
+ARGV.clear
+command = args.shift.strip rescue 'help'
+
+Autoweb::Command.run(command, args)

data/commands/baidump3.rb

@@ -0,0 +1,80 @@
+#!/usr/bin/env ruby
+require 'autoweb/page' 
+require 'iconv'
+
+include Autoweb
+
+Page.define "BaiduMp3" do |page|
+  page.url_tpl = 'http://mp3.baidu.com/m?f=3&rf=idx&tn=baidump3&ct=134217728&lf=&rn=&word=#{word}&lm=-1&oq=go&rsp=1'
+  
+  page.subs "result", "#Tbs tr" do |sub|
+    sub.ele "music", "td:nth(1) a"
+    sub.ele "artist", "td:nth(2) a"
+    sub.ele "album", "td:nth(3) a"
+    sub.ele "lyrics", "td:nth(5) a"
+    sub.ele "size", "td:nth(7)"
+    sub.ele "format", "td:nth(8)"
+  end
+end
+
+module Autoweb
+  module Command
+
+    class Baidump3 < Autoweb::Command::Base
+      MEGABYTE = 1024.0 * 1024.0
+      attr_accessor :dest_dir, :word
+   
+      def index
+        @word = args[0]
+        @dest_dir = args[1]
+        
+        if @word.nil?
+          return usage
+        end
+        
+        if @dest_dir
+          if File.directory?(dest_dir)
+            @dest_dir = File.expand_path(dest_dir)+"/"
+          else
+            error "#{@dest_dir} is not directory"
+          end
+        end
+        
+        search
+      end
+  
+      def search 
+        display("searching...")
+        page = Page.pages["BaiduMp3"].parse(:word => word)
+        result = page["result"][1]
+        music_url = result["music"][:href]
+        mp3url = decode open(URI.encode music_url).read[/var encurl = "([^"]*)"/,1]
+        display2("ok, parsing mp3...")
+        #size = `curl -I #{mp3url} 2>/dev/null`[/Content-Length:\ (\d+)/,1]
+        #display2(", size: %.2fM. " % (Integer(size)/MEGABYTE))
+        confirm("sure to download?") do
+          download_mp3(mp3url, word, result["format"].innerText)
+        end
+      end
+  
+      def usage
+        display "usage: autoweb mp3search MUSIC_NAME"
+      end
+  
+      def download_mp3(url, word, format)
+        `wget #{url} -O #{dest_dir}#{word.gsub(/[\+|\ |_]/, "-")}.#{format}`
+      end
+  
+      def decode(s)  
+        s.tr(_mktab(s[0].chr), s=~ /....:\// ? _mktab('h') : _mktab('f')) #http|ftp  
+      end
+
+      def _mktab(x)  
+        t0 = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"  
+        p = t0.partition(x)  
+        p[1] + p[2] + p[0]  
+      end
+  
+    end
+  end
+end

data/commands/help.rb

@@ -0,0 +1,16 @@
+module Autoweb::Command
+  class Help < Base  
+    def index
+      display usage
+    end
+
+    def usage
+      <<-EOTXT
+=== Command List:
+  autoweb baidump3 $music-name $save-to-dir
+ 
+
+EOTXT
+    end
+  end
+end

data/lib/autoweb.rb

@@ -0,0 +1,5 @@
+require 'hpricot'
+require 'open-uri'
+require 'ostruct'
+require 'active_support'
+require "autoweb/core.rb"

data/lib/autoweb/command.rb

@@ -0,0 +1,77 @@
+require 'autoweb/ui'
+module Autoweb
+  module Command
+    class Base
+      include Autoweb::UI
+      attr_accessor :args   
+      
+      def initialize(args)
+        @args = args             
+      end          
+      
+      def usage
+      <<-EOTXT
+=== Command List:
+  automan console
+  automan dbconsole
+  automan help
+  automan update
+ 
+
+EOTXT
+    	end
+      
+    end
+    class InvalidCommand < RuntimeError; end
+    class CommandFailed  < RuntimeError; end
+
+    class << self
+      
+      include Autoweb::UI
+      def run(command, args, retries=0)
+        begin         
+          run_internal(command, args.dup)
+        rescue InvalidCommand
+          error "Unknown command. Run 'autoweb help' for usage information."
+        rescue CommandFailed => e
+          error e.message
+        rescue Interrupt => e
+          error "\n[canceled]"
+        end
+      end
+
+      def run_internal(command, args)
+        klass, method = parse(command)
+        runner = klass.new(args)
+        raise InvalidCommand unless runner.respond_to?(method)
+        runner.send(method)
+      end
+
+      def parse(command)
+        parts = command.split(':')
+        case parts.size
+          when 1
+            begin
+              return eval("Autoweb::Command::#{command.capitalize}"), :index
+            #rescue NameError, NoMethodError
+            #  return Autoweb::Command::Help, command
+            end
+          when 2
+            begin
+              return Autoweb::Command.const_get(parts[0].capitalize), parts[1]
+            rescue NameError
+              raise InvalidCommand
+            end
+          else
+            raise InvalidCommand
+        end
+      end
+    end
+  end
+end
+
+Dir["#{File.dirname(__FILE__)}/../../commands/*.rb"].each { |c| 
+	unless (/_helper\.rb$/=~c)
+		require c 
+	end
+}

data/lib/autoweb/page.rb

@@ -0,0 +1,145 @@
+#!/usr/bin/env ruby
+require "ostruct"
+require "open-uri"
+require "hpricot"
+module Autoweb
+  module Container
+    attr_reader :name,  :sub_pages, :elements
+
+    def subs(name,sub_css,&block)
+     def_sub(name, sub_css, true, &block)
+    end
+
+    def sub(name,sub_css,&block)
+     def_sub(name, sub_css, false, &block)
+    end  
+
+    def ele(name, css)
+      @elements[name] = Element.new(self,name,css)
+    end
+
+    private
+
+    def def_sub(name, sub_css, is_array, &block)
+      sub = SubPage.new(self,name,sub_css,is_array)
+      yield sub
+      @sub_pages[name] = sub
+    end
+  end
+
+  class SubPage
+    attr_reader :parent, :css, :is_array
+    include Container  
+    def initialize(parent, name, css, is_array=false)
+      @parent = parent
+      @css = css
+      @name = name
+      @sub_pages = {}
+      @elements  = {}
+      @is_array = is_array
+    end 
+  end
+
+  class Element
+    attr_reader :parent, :name, :css
+    def initialize(parent,name, css)
+      @parent = parent
+      @name = name
+      @css = css
+    end
+  end
+
+  class Page
+    class << self
+      def pages
+        @pages||={}
+      end
+    end
+    attr_accessor :name, :url_tpl
+    include Container
+
+    def initialize(name)
+      @name = name
+      @sub_pages = {}
+      @elements  = {}
+    end
+
+    def self.define(name,&block)
+      page = self.new(name)
+      yield page
+      self.pages[name] = page
+      page
+    end
+
+    def parse(hash) 
+      Parser.new(self,hash).go
+    end
+
+    def url(locals) 
+      OpenStruct.new(locals.merge(:url_tpl=>self.url_tpl)).instance_eval{
+        eval %Q{"#{url_tpl.gsub(/"/, '\"')}"}
+      }
+    end
+  end
+
+  class Parser
+    attr_accessor :page, :url
+    def initialize(page, hash)
+      @page = page
+      if hash.is_a?(String) 
+        @url = hash
+      else
+        @url = page.url(hash)
+      end
+    end
+
+    def go
+      contaent_parser
+      self
+    end
+
+    def [](key)
+      contaent_parser[key]
+    end
+
+    def contaent_parser
+      @content_parser||=ContentParser.new(doc,page)
+    end
+
+    def doc
+      @doc||=Hpricot(open(URI.encode url))
+    end
+
+  end
+
+  class ContentParser
+    attr_accessor :doc, :page
+
+    def initialize(doc, page)
+      @doc = doc
+      @page = page
+    end
+
+    def [](key)
+      if r = page.sub_pages[key]
+        if r.is_array
+          doc.search(r.css).map{|e|self.class.new(e,r)}
+        else
+          self.new(doc.at(page.css),r)
+        end
+      elsif r = page.elements[key]
+         doc.at(r.css)
+      else
+        raise "key not found" 
+      end
+    end
+
+    def method_missing(key, *args, &block)
+      if doc.respond_to?(key)
+        doc.send key, *args, &block
+      else
+        super
+      end
+    end
+  end
+end

data/lib/autoweb/ui.rb

@@ -0,0 +1,89 @@
+module Autoweb
+  module UI
+    class RetryError < RuntimeError
+    end 
+    
+    class Input < String
+      include UI
+      def process_default(options={})
+        if is?("q")
+          exit(0)
+        elsif is?("h")||is?("?")
+          display options[:help]||"no help"
+          raise RetryError
+        else
+          raise RetryError
+        end
+      end
+      
+      
+      def yes?
+        self.is?("y")
+      end
+      
+      def is?(str)
+        return if str.nil?
+        self.downcase.strip == str.downcase.strip
+      end
+    end    
+    
+    def display(msg, new_line = true)
+      msg = msg.to_s.gsub(/_/) { ' ' } 
+      if new_line
+        STDOUT.puts msg
+      else
+        STDOUT.print msg
+      end
+      STDOUT.flush
+    end
+    
+    def error(msg="error")
+      display msg
+      exit(1)
+    end
+    
+    def display2(msg)
+      display(msg,false)
+    end
+    
+    def confirm(message=nil, options={})
+      if message.nil?
+        message = "Are you sure you wish to continue?"
+      end
+      
+      message << "(y/q/h)"
+      
+      ask_loop(message) do |input|
+        if input.yes?
+          yield
+        else
+          input.process_default(:help=>options[:help])
+        end
+      end
+      ask.downcase == 'y'
+    end
+
+    def format_date(date)
+      date = Time.parse(date) if date.is_a?(String)
+      date.strftime("%Y-%m-%d %H:%M %Z")
+    end
+    
+    def ask_loop(message,&block)
+      display2 message+" "
+      begin
+        yield Input.new(ask)
+      rescue RetryError => e
+        display2 message
+        retry      
+      end
+    end
+    
+    def ask
+      gets.strip
+    end
+
+    def shell(cmd)
+      FileUtils.cd(Dir.pwd) {|d| return `#{cmd}`}
+    end
+  end
+end

data/lib/autoweb/version.rb

@@ -0,0 +1,3 @@
+module Autoweb
+  VERSION = '0.0.2'
+end

metadata

@@ -0,0 +1,72 @@
+--- !ruby/object:Gem::Specification 
+name: autoweb
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 2
+  version: 0.0.2
+platform: ruby
+authors: 
+- dazuiba
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-08-07 00:00:00 +08:00
+default_executable: 
+dependencies: []
+
+description: Automate the Internet. baidu music downloader
+email: come2u@gmail.com
+executables: 
+- autoweb
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- README.md
+- bin/autoweb
+- lib/autoweb/command.rb
+- lib/autoweb/page.rb
+- lib/autoweb/ui.rb
+- lib/autoweb/version.rb
+- lib/autoweb.rb
+- commands/baidump3.rb
+- commands/help.rb
+has_rdoc: true
+homepage: http://github.com/dazuiba/autoweb.git
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 1
+      - 8
+      - 7
+      version: 1.8.7
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: Gem for the rest
+test_files: []
+