atig 0.0.1

data/lib/atig/ofilter.rb

@@ -0,0 +1,4 @@
+require 'atig/ofilter/escape_url'
+require 'atig/ofilter/short_url'
+require 'atig/ofilter/geo'
+require 'atig/ofilter/footer'

data/lib/atig/ofilter/escape_url.rb

@@ -0,0 +1,102 @@
+# -*- mode:ruby; coding:utf-8 -*-
+
+require 'atig/util'
+require 'atig/http'
+require 'atig/url_escape'
+
+begin
+  require "punycode"
+rescue LoadError
+end
+
+module Atig
+  module OFilter
+    class EscapeUrl
+      include Util
+      def initialize(context)
+        @log  = context.log
+        @http = Atig::Http.new @log
+      end
+
+      def call(status)
+        status.merge(:status => escape_http_urls(status[:status]))
+      end
+
+      def exist_uri?(uri, limit = 1)
+        ret = nil
+        #raise "Not supported." unless uri.is_a?(URI::HTTP)
+        return ret if limit.zero? or uri.nil? or not uri.is_a?(URI::HTTP)
+        @log.debug uri.inspect
+
+        req = @http.req :head, uri
+        @http.http(uri, 3, 2).request(req) do |res|
+          ret = case res
+                when Net::HTTPSuccess
+                true
+                when Net::HTTPRedirection
+                uri = resolve_http_redirect(uri)
+                  exist_uri?(uri, limit - 1)
+                when Net::HTTPClientError
+                  false
+                else
+                  nil
+                end
+        end
+        ret
+      rescue => e
+        @log.error e.inspect
+        ret
+      end
+
+      def escape_http_urls(text)
+        original_text = text.encoding!("UTF-8").dup
+
+        if defined? ::Punycode
+          # TODO: Nameprep
+          text.gsub!(%r{(https?://)([^\x00-\x2C\x2F\x3A-\x40\x5B-\x60\x7B-\x7F]+)}) do
+            domain = $2
+            # Dots:
+            #   * U+002E (full stop)           * U+3002 (ideographic full stop)
+            #   * U+FF0E (fullwidth full stop) * U+FF61 (halfwidth ideographic full stop)
+            # => /[.\u3002\uFF0E\uFF61] # Ruby 1.9 /x
+            $1 + domain.split(/\.|\343\200\202|\357\274\216|\357\275\241/).map do |label|
+              break [domain] if /\A-|[\x00-\x2C\x2E\x2F\x3A-\x40\x5B-\x60\x7B-\x7F]|-\z/ === label
+              next label unless /[^-A-Za-z0-9]/ === label
+              punycode = Punycode.encode(label)
+              break [domain] if punycode.size > 59
+              "xn--#{punycode}"
+            end.join(".")
+          end
+          if text != original_text
+            log :info, "Punycode encoded: #{text}"
+            original_text = text.dup
+          end
+        end
+
+        urls = []
+        text.split(/[\s<>]+/).each do |str|
+          next if /%[0-9A-Fa-f]{2}/ === str
+          # URI::UNSAFE + "#"
+          escaped_str = URI.escape(str, %r{[^-_.!~*'()a-zA-Z0-9;/?:@&=+$,\[\]#]}) #'
+          URI.extract(escaped_str, %w[http https]).each do |url|
+            uri = URI(URI.rstrip(url))
+            if not urls.include?(uri.to_s) and self.exist_uri?(uri)
+              urls << uri.to_s
+            end
+          end if escaped_str != str
+        end
+        urls.each do |url|
+          unescaped_url = URI.unescape(url).encoding!("UTF-8")
+          text.gsub!(unescaped_url, url)
+        end
+        log :info, "Percent encoded: #{text}" if text != original_text
+
+        text.encoding!("UTF-8")
+      rescue => e
+        log :error, e
+        text
+      end
+    end
+  end
+end
+

data/lib/atig/ofilter/footer.rb

@@ -0,0 +1,20 @@
+# -*- mode:ruby; coding:utf-8 -*-
+
+module Atig
+  module OFilter
+    class Footer
+      def initialize(context)
+        @opts = context.opts
+      end
+
+      def call(q)
+        if @opts.footer && !@opts.footer.empty? then
+          q.merge :status => "#{q[:status]} #{@opts.footer}"
+        else
+          q
+        end
+
+      end
+    end
+  end
+end

data/lib/atig/ofilter/geo.rb

@@ -0,0 +1,17 @@
+# -*- mode:ruby; coding:utf-8 -*-
+
+module Atig
+  module OFilter
+    class Geo
+      def initialize(context)
+        @opts = context.opts
+      end
+
+      def call(q)
+        return q unless @opts.ll
+        lat, long = @opts.ll.split(",", 2)
+        q.merge :lat  => lat.to_f, :long => long.to_f
+      end
+    end
+  end
+end

data/lib/atig/ofilter/short_url.rb

@@ -0,0 +1,47 @@
+# -*- mode:ruby; coding:utf-8 -*-
+
+require 'atig/unu'
+require 'atig/bitly'
+
+module Atig
+  module OFilter
+    class ShortUrl
+      MIN_LEN = 20
+
+      def initialize(context)
+        @log  = context.log
+        @opts = context.opts
+        @http = Atig::Http.new @log
+      end
+
+      def call(status)
+        mesg = status[:status]
+        status.merge(:status => short_urls(mesg))
+      end
+
+      def short_urls(mesg)
+        shorten = case
+                  when @opts.bitlify.to_s.include?(":")
+                    login, key, len = @opts.bitlify.to_s.split(":", 3)
+                    @len = (len || MIN_LEN).to_i
+                    Bitly.login @log, login, key
+                  when @opts.bitlify
+                    @len = (@opts.bitlify.to_s || MIN_LEN).to_i
+                    Bitly.no_login @log
+                  when @opts.unuify
+                    @len = (@opts.unuify.to_s || MIN_LEN).to_i
+                    Unu.new @log
+                  else
+                    return mesg
+                  end
+        mesg.gsub(URI.regexp(%w[http https])) do|url|
+          if URI.rstrip(url).size < @len then
+            url
+          else
+            shorten.shorten url
+          end
+        end
+      end
+    end
+  end
+end

data/lib/atig/option.rb

@@ -0,0 +1,90 @@
+# -*- coding: utf-8 -*-
+module Atig
+  class Option
+    class << self
+      def default_value(name, value)
+        @default ||= {}
+        @default[name.to_sym] = value
+      end
+
+      def parse(str)
+        @default ||= {}
+        opts = str.split(" ")
+        opts = opts.inject({}) do |r, i|
+          key, value = i.split("=", 2)
+
+          r.update key.to_sym => parse_value(value)
+        end
+        self.new(@default.merge(opts))
+      end
+
+      def parse_value(value)
+        case value
+        when nil, /\Atrue\z/          then true
+        when /\Afalse\z/              then false
+        when /\A\d+\z/                then value.to_i
+        when /\A(?:\d+\.\d*|\.\d+)\z/ then value.to_f
+        else                               value
+        end
+      end
+    end
+
+    default_value :api_base,        'https://api.twitter.com/1/'
+    default_value :stream_api_base, 'http://chirpstream.twitter.com/2b/'
+    default_value :search_api_base, 'http://search.twitter.com/'
+
+    def initialize(table)
+      @table = {}
+      table.each do|key,value|
+        key = key.to_sym
+        @table[key] = value
+        new_ostruct_member key
+      end
+
+      # Ruby 1.8だとidというフィールドが作れないので作っておく
+      new_ostruct_member :id, true
+    end
+
+    def marshal_dump; @table end
+    def fields;
+      @table.keys
+    end
+
+    def [](name)
+      @table[name.to_sym]
+    end
+
+    def []=(name, value)
+      @table[name.to_sym] = value
+    end
+
+    def new_ostruct_member(name, force=false)
+      if force or (not self.respond_to?(name))
+        class << self; self; end.class_eval do
+          define_method(name) { @table[name] }
+          define_method(:"#{name}=") { |x| @table[name] = x }
+        end
+      end
+    end
+
+    def method_missing(mid, *args) # :nodoc:
+      mname = mid.id2name
+      len = args.length
+      if mname =~ /=$/
+        if len != 1
+          raise ArgumentError, "wrong number of arguments (#{len} for 1)", caller(1)
+        end
+        if self.frozen?
+          raise TypeError, "can't modify frozen #{self.class}", caller(1)
+        end
+        name = mname.chop.to_sym
+        @table[name] = args[0]
+        self.new_ostruct_member(name)
+      elsif len == 0
+        @table[mid]
+      else
+        raise NoMethodError, "undefined method `#{mname}' for #{self}", caller(1)
+      end
+    end
+  end
+end

data/lib/atig/scheduler.rb

@@ -0,0 +1,79 @@
+# -*- mode:ruby; coding:utf-8 -*-
+
+require 'atig/util'
+
+module Atig
+  class Scheduler
+    include Util
+
+    attr_reader :search
+
+    def initialize(context, api, search, stream)
+      @log    = context.log
+      @api    = api
+      @search = search
+      @stream = stream
+      @agents = []
+
+      @queue = SizedQueue.new 10
+      daemon do
+        f = @queue.pop
+        f.call
+      end
+    end
+
+    def repeat(interval,opts={}, &f)
+      @queue.push(lambda{ safe { f.call @api } })
+      t = daemon do
+        sleep interval
+        log :debug, "agent #{t.inspect} is invoked"
+        @queue.push(lambda{ safe { f.call @api } })
+      end
+
+      log :info, "repeat agent #{t.inspect} is registered"
+      @agents << t
+      t
+    end
+
+    def delay(interval, opts={}, &f)
+      sleep interval
+      @queue.push(lambda{ safe { f.call @api } })
+    end
+
+    def stream(&f)
+      return nil unless @stream
+      @stream_thread.kill if @stream_thread
+      @stream_thread = daemon {
+        f.call @stream
+      }
+    end
+
+    def re_try(count, &f)
+      begin
+        f.call
+      rescue => e
+        log :error, [count, e.inspect].inspect
+        if count > 0
+          count -= 1
+          sleep 1
+          log :debug, "retry"
+          retry
+        end
+        log :error, "Some Error Happened: #{e}"
+        raise e
+      end
+    end
+
+    def limit
+      @api.limit
+    end
+
+    def remain
+      @api.remain
+    end
+
+    def reset
+      @api.reset
+    end
+  end
+end

data/lib/atig/search.rb

@@ -0,0 +1,22 @@
+require 'rubygems'
+require 'json'
+require 'atig/http'
+require 'atig/url_escape'
+
+module Atig
+  class Search
+    def search(query, options = {})
+      search = URI("http://search.twitter.com/search.json")
+      search.path = "/search.json"
+      params = options; options[:q] = query
+      search.query = options.to_query_str
+      http = Http.new nil
+      req = http.req(:get, search)
+      res = http.http(search, 5, 10).request(req)
+      res = JSON.parse(res.body)
+    rescue Errno::ETIMEDOUT, JSON::ParserError, IOError, Timeout::Error, Errno::ECONNRESET => e
+      @log.error e
+      text
+    end
+  end
+end

data/lib/atig/search_twitter.rb

@@ -0,0 +1,21 @@
+# -*- mode:ruby; coding:utf-8 -*-
+require 'atig/basic_twitter'
+require 'atig/http'
+
+module Atig
+  # from tig.rb
+  class SearchTwitter < BasicTwitter
+    def initialize(context)
+      super context, :search_api_base
+      @http  = Atig::Http.new @log
+    end
+
+    protected
+    def request(uri, opts)
+      header      = {}
+      req         = @http.req :get, uri, header
+      @log.debug [req.method, uri.to_s]
+      @http.http(uri, 30, 30).request req
+    end
+  end
+end

data/lib/atig/sized_hash.rb

@@ -0,0 +1,33 @@
+# -*- mode:ruby; coding:utf-8 -*-
+require 'forwardable'
+
+module Atig
+  class SizedHash
+    extend Forwardable
+
+    def_delegators :@xs, :size
+
+    def initialize(size)
+      @size = size
+      @xs   = []
+    end
+
+    def [](key)
+      kv = @xs.assoc(key)
+      if kv then
+        @xs.delete kv
+        @xs.push   kv
+        kv[1]
+      end
+    end
+
+    def []=(key,value)
+      @xs.push [key, value]
+      @xs.shift if @xs.size > @size
+    end
+
+    def key?(key)
+      @xs.any?{|k,_| key == k }
+    end
+  end
+end