croudia 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in croudia.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 wktk
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Croudia
+
+A Mechanize-based Ruby scraper for [Croudia](https://croudia.com)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'croudia'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install croudia
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/croudia.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/croudia/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ['wktk']
+  gem.email         = ['wktk@wktk.in']
+  gem.description   = 'A Mechanize-based Ruby scraper for Croudia'
+  gem.summary       = gem.description
+  gem.homepage      = 'https://github.com/wktk/croudia-gem'
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = 'croudia'
+  gem.require_paths = ['lib']
+  gem.version       = Croudia::VERSION
+
+  gem.add_dependency 'mechanize', '~> 2.5.1'
+  gem.add_development_dependency 'rspec', '~> 2.10.1'
+  gem.add_development_dependency 'webmock', '~> 1.8.7'
+end

data/lib/croudia/api.rb

@@ -0,0 +1,5 @@
+module Croudia
+  module API
+    # Fingers crossed they will soon release their own ( '-^)b
+  end
+end

data/lib/croudia/identity.rb

@@ -0,0 +1,39 @@
+module Croudia
+  class Identity
+    attr_reader :id, :time
+    alias id_str id
+    alias created_at time
+
+    def initialize(attrs)
+      attrs.keys.each do |key|
+        if 'time' == key.to_s && !attrs[key].is_a?(Time)
+          if /^\d+$/ =~ attrs[key]
+            attrs[key] = Time.at(attrs[key].to_i)
+          else
+            attrs[key] = Time.new(*attrs[key].split(/\D/))
+          end
+        elsif /_count$/ =~ key.to_s
+          attrs[key] = attrs[key].to_i
+        end
+
+        instance_variable_set(:"@#{key}", attrs[key])
+      end
+    end
+
+    def ==(other)
+      super || self.class == other.class && @id == other.id
+    end
+
+    def [](name)
+      instance_variable_get(:"@#{name}")
+    end
+
+    def id
+      @id
+    end
+
+    def method_missing(name, *args)
+      self[name]
+    end
+  end
+end

data/lib/croudia/scraper/friendships.rb

@@ -0,0 +1,33 @@
+module Croudia
+  class Scraper
+    module Friendships
+      def follow(username)
+        require_login
+        post('/follows/target_follow', 
+          :target_user_name => username,
+          :follow_action => 'enable',
+        )
+      end
+
+      def unfollow(username)
+        require_login
+        post('/follows/target_follow', 
+          :target_user_name => username,
+          :follow_action => 'diable', # Yes, it is!
+        )
+      end
+
+      def following(username=nil, params={})
+        username = @current_user.username unless username
+        Parser.user_list(get("/follows/following/#{username}", params))
+      end
+      alias followings following
+
+      def follower(username=nil, params={})
+        username = @current_user.username unless username
+        Parser.user_list(get("/follows/follower/#{username}", params))
+      end
+      alias followers follower
+    end
+  end
+end

data/lib/croudia/scraper/login.rb

@@ -0,0 +1,40 @@
+module Croudia
+  class Scraper
+    module Login
+      def login(username, password)
+        get('/')
+        params = {
+          :username => username,
+          :password => password,
+          :auto_login => 'on',
+        }
+        post('/', params)
+        raise ArgumentError, 'Wrong ID or password' if not logged_in?
+      end
+      alias login! login
+
+      def logout
+        require_login
+        post('/login/logout', :method_name => 'logout')
+        @current_user = nil
+        @logged_in = false
+      end
+      alias logout! logout
+
+      def logged_in?(force=false)
+        @logged_in = nil if force
+        @logged_in = @logged_in.nil? ? '/' != get('/').uri.path : @logged_in
+      end
+      alias logged_in logged_in?
+
+      def logged_out?(force=false)
+        !logged_in?(force)
+      end
+      alias logged_out logged_out?
+
+      def require_login
+        raise 'Not logged in' if logged_out?
+      end
+    end
+  end
+end

data/lib/croudia/scraper/parser/users.rb

@@ -0,0 +1,53 @@
+module Croudia
+  class Scraper
+    module Parser
+      module Users
+        def user_page(page)
+          return nil if /on Croudia$/ !~ page.title
+          doc = page.parser
+          attrs = {
+            :username => doc.xpath('//a[contains(@href, "/voices/user/")]').first.attribute('href').content.sub(%r{^/voices/user/}, ''),
+            :nickname => doc.css('.avatar_l').first.attribute('alt').content,
+            :avatar => "#{page.uri.scheme}://#{page.uri.host}" + doc.css('.avatar_l').first.attribute('src').content,
+            :self_introduction => doc.css('.fontg').first.content.strip,
+            :voices_count => doc.xpath('//span[preceding-sibling::a[contains(@href,"/voices/user/")]]').first.content,
+            :following_count => doc.xpath('//span[preceding-sibling::a[contains(@href,"/following/")]]').first.content,
+            :followers_count => doc.xpath('//span[preceding-sibling::a[contains(@href,"/follower/")]]').first.content,
+            :album_count => doc.xpath('//span[preceding-sibling::a[contains(@href,"/album")]]').first.content,
+            :spreadia => doc.xpath('//span[preceding-sibling::a[contains(@href,"/spreadia/")]]').first.content,
+            :favodia => doc.xpath('//span[preceding-sibling::a[contains(@href,"/favodia/")]]').first.content,
+          }
+          if (location = doc.xpath('//a[contains(@href, "/users/google_maps")]')).size.nonzero?
+            attrs[:position] = location.first.content
+          end
+          if (url = doc.xpath('//a[@target="_blank"][name(..)="li"]')).size.nonzero?
+            attrs[:user_url] = url.first.content
+          end
+          if (friendships = doc.css('.font1')).size.nonzero?
+            attrs[:following] = /#e/i !~ friendships[0].attribute('style').value
+            attrs[:followed_by] = /#e/i !~ friendships[1].attribute('style').value
+          end
+          Croudia::User.new(attrs)
+        end
+
+        def user_list(page, klass=Croudia::User)
+          user_nodes = page.parser.xpath('//div[@data-role="content"]/ul[@data-role="listview"]/li[child::a]')
+          users = []
+          user_nodes.map do |user_node|
+            attrs = {
+              :username => user_node.css('a').first.attribute('href').value.sub(%r{^/}, ''),
+              :nickname => user_node.css('.bord').first.content,
+              :avatar => "#{page.uri.scheme}://#{page.uri.host}" + user_node.css('img').first.attribute('src').value,
+              :self_introduction => user_node.css('.fontg').first.content.strip,
+            }
+            if (following = user_node.css('.font1')).size.nonzero?
+              attrs[:following] = /#e/i !~ following.first.attribute('style').value
+            end
+            users << Croudia::User.new(attrs)
+          end
+          users
+        end
+      end
+    end
+  end
+end

data/lib/croudia/scraper/parser/voices.rb

@@ -0,0 +1,32 @@
+module Croudia
+  class Scraper
+    module Parser
+      module Voices
+        def voices(page)
+          voice_nodes = page.parser.xpath('//div[@data-role="content"]/ul[@data-role="listview"]/li[child::a]')
+          voices = []
+          voice_nodes.each do |voice|
+            user = Croudia::User.new(
+              :username => voice.xpath('//a[parent::span[@class="test_span"]]').first.attribute('href').value.gsub(/\W/, ''),
+              :nickname => voice.xpath('//span[@class="bord"][parent::p]').first.content,
+              :avatar => "#{page.uri.scheme}://#{page.uri.host}" + voice.xpath('//img[@class="avatar"]').attribute('src').value,
+            )
+            attrs = {
+              :id => voice.xpath('//a[contains(@href,"/voices/show/")]').first.attribute('href').value.gsub(/\D/, ''),
+              :user => user,
+              :voice_desc => voice.css('.voice_desc').first.content,
+              :spreaded_count => voice.xpath('//span[contains(@style,"#396")]').first.content.gsub(/\D/, ''),
+              :favorited_count => voice.xpath('//span[contains(@style,"520")]').first.content.gsub(/\D/, ''),
+            }
+            if (time = voice.xpath('//p[@data-time]')).size.nonzero?
+              # For timeline
+              attrs[:time] = time.first.attribute('data-time').value
+            end
+            voices << Croudia::Voice.new(attrs)
+          end
+          voices
+        end
+      end
+    end
+  end
+end

data/lib/croudia/scraper/parser.rb

@@ -0,0 +1,13 @@
+require 'croudia/scraper/parser/users'
+require 'croudia/scraper/parser/voices'
+
+module Croudia
+  class Scraper
+    module Parser
+      class << self
+        include Croudia::Scraper::Parser::Users
+        include Croudia::Scraper::Parser::Voices
+      end
+    end
+  end
+end

data/lib/croudia/scraper/users.rb

@@ -0,0 +1,20 @@
+module Croudia
+  class Scraper
+    module Users
+      def current_user(force=false)
+        require_login
+        @current_user = nil if force
+        @current_user ||= Parser.user_page(get('/users/redirect_user'))
+      end
+      alias verify_credentials current_user
+
+      def user(username=nil, params={})
+        if username
+          Parser.user_page(get("/#{username}", params))
+        else
+          current_user
+        end
+      end
+    end
+  end
+end

data/lib/croudia/scraper/voices.rb

@@ -0,0 +1,27 @@
+module Croudia
+  class Scraper
+    module Voices
+      def timeline(params={})
+        require_login
+        Parser.voices(get('/voices/timeline', params))
+      end
+      alias home_timeline timeline
+
+      def reply_list(params={})
+        require_login
+        Parser.voices(get('/voices/reply_list', params))
+      end
+      alias mentions reply_list
+      alias mentions_timeline reply_list
+
+      def update(text)
+        require_login
+        page = get('/voices/written')
+        form = page.form_with(:action => '/voices/write')
+        form['voice[tweet]'] = text
+        result = form.submit
+        raise 'Update failed' if result.body.include? 'error_popup'
+      end
+    end
+  end
+end

data/lib/croudia/scraper.rb

@@ -0,0 +1,62 @@
+require 'croudia/scraper/friendships'
+require 'croudia/scraper/login'
+require 'croudia/scraper/parser'
+require 'croudia/scraper/users'
+require 'croudia/scraper/voices'
+require 'croudia/user'
+require 'croudia/version'
+require 'croudia/voice'
+require 'mechanize'
+
+module Croudia
+  class Scraper
+    include Croudia::Scraper::Friendships
+    include Croudia::Scraper::Login
+    include Croudia::Scraper::Users
+    include Croudia::Scraper::Voices
+
+    attr_accessor :agent, :endpoint
+
+    def initialize(username=nil, password=nil)
+      yield self if block_given?
+
+      @endpoint ||= 'https://croudia.com'
+      @user_agent ||= "The Croudia Gem/#{Croudia::VERSION} (https://github.com/wktk/croudia-gem)"
+      @agent ||= Mechanize.new do |s|
+        s.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      end
+      self.user_agent = @user_agent
+
+      username && password && login(username, password)
+    end
+
+    def user_agent
+      @agent.user_agent
+    end
+
+    def user_agent=(ua)
+      unless @agent
+        @user_agent = ua
+      else
+        @agent.user_agent = ua
+      end
+    end
+
+    def get(path, params={})
+      pick_token(@agent.get(@endpoint + path, params))
+    end
+
+    def post(path, params={})
+      params = { :authenticity_token => @authenticity_token }.merge(params)
+      pick_token(@agent.post(@endpoint + path, params))
+    end
+
+  private
+    def pick_token(page)
+      page.respond_to?('parser') &&
+        (token = page.parser.xpath('//meta[@name="csrf-token"]')).size.nonzero? &&
+          @authenticity_token = token.first.attribute('content').content
+      page
+    end
+  end
+end

data/lib/croudia/user.rb

@@ -0,0 +1,43 @@
+require 'croudia/identity'
+
+module Croudia
+  class User < Croudia::Identity
+    attr_reader :username, :nickname, :avatar, :self_introduction,
+      :voices_count, :following_count, :followers_count, :album_count,
+      :position, :user_url, :spreadia, :favodia,
+      :following, :followed_by
+    alias handle username
+    alias screen_name username
+    alias user_name username
+    alias name nickname
+    alias profile_image avatar
+    alias description self_introduction
+    alias voice_count voices_count
+    alias statuses_count voices_count
+    alias status_count voices_count
+    alias updates_count voices_count
+    alias update_count voices_count
+    alias followings_count following_count
+    alias friends_count following_count
+    alias friend_count following_count
+    alias follower_count followers_count
+    alias location position
+    alias url user_url
+    alias favorited_count favodia
+    alias favourited_count favodia
+    alias spreaded_count spreadia
+    alias following? following
+    alias followed_by? followed_by
+
+    def initialize(*args)
+      super
+      @id = @username
+      @spreadia = @spreadia.to_i if @spreadia
+      @favodia = @favodia.to_i if @favodia
+    end
+
+    def to_s
+      @username
+    end
+  end
+end

data/lib/croudia/version.rb

@@ -0,0 +1,3 @@
+module Croudia
+  VERSION = '0.0.1'
+end

data/lib/croudia/voice.rb

@@ -0,0 +1,21 @@
+require 'croudia/identity'
+
+module Croudia
+  class Voice < Croudia::Identity
+    attr_reader :user, :voice_desc, :spreaded_count, :favorited_count, :time
+    alias from_user user
+    alias text voice_desc
+    alias description voice_desc
+    alias desc voice_desc
+    alias spread_count spreaded_count
+    alias favorite_count favorited_count
+
+    def to_s
+      "@#{@user.username}: #{@desc}"
+    end
+
+    def to_i
+      @id.to_i
+    end
+  end
+end

data/lib/croudia.rb

@@ -0,0 +1,23 @@
+require 'croudia/scraper'
+
+module Croudia
+  class << self
+    # Delegate to Croudia::Scraper.new
+    #
+    # @return [Croudia::Scraper]
+    def scraper(*args, &block)
+      Croudia::Scraper.new(*args, &block)
+    end
+    alias new scraper
+    alias client scraper
+
+    def method_missing(name, *args, &block)
+      return super unless scraper.respond_to?(name)
+      scraper.send(name, *args, &block)
+    end
+
+    def respond_to?(name)
+      scraper.respond_to?(name) || super
+    end
+  end
+end

data/spec/croudia/api_spec.rb

@@ -0,0 +1,7 @@
+require 'croudia/api'
+
+describe Croudia::API do
+  it 'is a Module' do
+    Croudia::API.should be_a Module
+  end
+end

data/spec/croudia/identity_spec.rb

@@ -0,0 +1,65 @@
+require 'helper'
+
+describe Croudia::Identity do
+  describe '#initialize' do
+    it 'sets arguments as instance variables' do
+      Croudia::Identity.new(:hello => 'hi').instance_variable_get('@hello').should eq 'hi'
+    end
+
+    it 'converts a Unix time attribution named time to a Time object' do
+      Croudia::Identity.new(:time => '1234567890').time.should be_a Time
+    end
+
+    it 'converts a date string named time to a Time object' do
+      Croudia::Identity.new(:time => '2013-01-01 11:55').time.should be_a Time
+    end
+
+    it 'converts *_count into an Integer' do
+      Croudia::Identity.new(:foo_count => '2012').foo_count.should eq 2012
+    end
+  end
+
+  describe '#==' do
+    it 'returns true when IDs match' do
+      obj1 = Croudia::Identity.new(:id => '1', :username => 'wktk')
+      obj2 = Croudia::Identity.new(:id => '1', :username => 'croudia')
+      (obj1 == obj2).should be_true
+    end
+
+    it 'returns false when IDs do not match' do
+      obj1 = Croudia::Identity.new(:id => '1', :username => 'wktk')
+      obj2 = Croudia::Identity.new(:id => '2', :username => 'wktk')
+      (obj1 == obj2).should be_false
+    end
+
+    it 'returns false when classes do not match' do
+      obj1 = Croudia::Identity.new(:id => '1')
+      obj2 = { :id => '1' }
+      (obj1 == obj2).should be_false
+    end
+  end
+
+  describe '#[]' do
+    it 'returns the same name instance variable' do
+      Croudia::Identity.new(:foo => 'bar')[:foo].should eq 'bar'
+    end
+  end
+
+  describe '#id' do
+    it 'returns @id, not object_id' do
+      identity = Croudia::Identity.new(:id => '1')
+      identity.id.should_not eq identity.object_id
+      identity.id.should eq identity.instance_variable_get('@id')
+    end
+  end
+
+  describe '#method_missing' do
+    it 'returns the same name instance variable if set' do
+      Croudia::Identity.new(:foo => 'bar').foo.should eq 'bar'
+    end
+
+    it 'returns nil if not set' do
+      Croudia::Identity.new(:foo => 'bar').bar.should be_nil
+    end
+  end
+end

data/spec/croudia/scraper/friendships_spec.rb

@@ -0,0 +1,61 @@
+require 'helper'
+
+describe Croudia::Scraper::Friendships do
+  context 'when logged in' do
+    before do
+      @croudia = Croudia::Scraper.new
+      @croudia.instance_variable_set('@logged_in', true)
+    end
+
+    describe '#follow' do
+      it 'posts to the correct resource' do
+        stub_post('/follows/target_follow').to_return(:headers => {:content_type => 'text/html'})
+        @croudia.follow('wktk')
+        a_post('/follows/target_follow').with(:body => /target_user_name=wktk/).should have_been_made
+      end
+    end
+
+    describe '#unfollow' do
+      it 'posts to the correct resource' do
+        stub_post('/follows/target_follow').to_return(:headers => {:content_type => 'text/html'})
+        @croudia.follow('wktk')
+        a_post('/follows/target_follow').with(:body => /target_user_name=wktk/).should have_been_made
+      end
+    end
+  end
+
+  context 'when not logged in' do
+    before do
+      @croudia = Croudia::Scraper.new
+      @croudia.instance_variable_set('@logged_in', true)
+    end
+
+    describe '#follow' do
+      it 'should raise error' do
+        expect{ @croudia.follow('wktk') }.to raise_error 
+      end
+    end
+
+    describe '#unfollow' do
+      it 'should raise error' do
+        expect{ @croudia.follow('wktk') }.to raise_error
+      end
+    end
+  end
+
+  describe '#follower' do
+    it 'gets the correct resource' do
+      stub_get('/follows/follower/wktk').to_return(:headers => {:content_type => 'text/html'})
+      Croudia.follower('wktk')
+      a_get('/follows/follower/wktk').should have_been_made
+    end
+  end
+
+  describe '#following' do
+    it 'gets the correct resource' do
+      stub_get('/follows/following/wktk').to_return(:headers => {:content_type => 'text/html'})
+      Croudia.following('wktk')
+      a_get('/follows/following/wktk').should have_been_made
+    end
+  end
+end