hatebu_entry 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0aab4957200e35089c390385b7783515754062e9
+  data.tar.gz: 34e4799ece62fabbfecaa37c488ed7a53a6b383d
+SHA512:
+  metadata.gz: 7a780f297fd288a18adfecbdf8fc17acd19c6f012fe129d15154229094468bb443ef1e77ac9a8ec608b0e17634a6e8cfbaa10590abfb96456eef27408cd78be5
+  data.tar.gz: 06a8e2c519b77497d4c88a48904670bef01b5bee8fed6ca31cd5548c0ac00ae2d581a313d190af8684e4a5ae0f082db4198781beb9ab0830a2c4b9902da5871b

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 2.0.0

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in hatebu_entry.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 kyoendo
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,52 @@
+# HatebuEntry
+
+HatebuEntry is a tool for retrieving and handling Hatena Bookmark entry lists written in Ruby.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'hatebu_entry'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install hatebu_entry
+
+## Usage
+
+Try this;
+
+```ruby
+require 'hatebu_entry'
+
+uri = 'http://d.hatena.ne.jp'
+hent = HatebuEntry.new(uri)
+
+puts hent.entries
+```
+
+You will get like this.
+
+     6691: 僕は自分が思っていたほどは頭がよくなかった - しのごの... (http://d.hatena.ne.jp/tictac/20120110/p1)
+     5788: 「 2 」か「 9 」で割ってみる - ナイトシフト (http://d.hatena.ne.jp/nightshift/20090121/1232521713)
+     5605: 読みやすい文章を書くための技法 - RyoAnna’s iPhone Blog (http://d.hatena.ne.jp/RyoAnna/20100824/1282660678)
+     4483: この「いじめ対策」はすごい！ - 森口朗公式ブログ (http://d.hatena.ne.jp/moriguchiakira/20090520)
+     4167: 知らないと損する英語の速読方法（1） - 一法律学徒の英語... (http://d.hatena.ne.jp/kousuke-i/20081203/1228314824)
+     3973: パワポでもここまでできる！米財務省から学べる美しい資... (http://d.hatena.ne.jp/stj064/20120401/p1)
+     3968: デジタル一眼レフカメラの基礎から実践まで - #RyoAnnaBlog (http://d.hatena.ne.jp/RyoAnna/20120501/1335884196)
+     3853: 『忙しい人』と『仕事ができる人』の２０の違い (http://d.hatena.ne.jp/favre21/20070927)
+     3717: MacBook Air 11インチ欲しい！とは - はてなキーワード (http://d.hatena.ne.jp/keyword/MacBook%20Air%2011%A5%A4%A5%F3%A5%C1%CD%DF%A4%B7%A4%A4%A1%AA)
+     3715: おさえておきたいメールで使う敬語 - かみんぐあうとっ (http://d.hatena.ne.jp/komoko-i/20110524/p1)
+
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/hatebu_entry

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'hatebu_entry'
+
+HatebuEntry::Command.start(ARGV)

data/hatebu_entry.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'hatebu_entry/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "hatebu_entry"
+  spec.version       = HatebuEntry::VERSION
+  spec.authors       = ["kyoendo"]
+  spec.email         = ["postagie@gmail.com"]
+  spec.description   = %q{HatebuEntry is a tool for retrieving and handling Hatena Bookmark entry lists written in Ruby.}
+  spec.summary       = %q{Hatena bookmark entry list handler}
+  spec.homepage      = "https://github.com/melborne/hatebu_entry"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.required_ruby_version = '>= 1.9.3'
+  spec.add_runtime_dependency 'nokogiri'
+  spec.add_runtime_dependency 'thor'
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "fakeweb", "~> 1.3"
+end

data/lib/hatebu_entry/command.rb

@@ -0,0 +1,47 @@
+require 'thor'
+
+class HatebuEntry::Command < Thor
+  desc "get URL", "Get Hatena bookmark entries for URL"
+  option :pages, aliases:"-p", default:0
+  option :sort, aliases:"-s", default:"count"
+  def get(url)
+    entries = HatebuEntry.new(url, options[:sort])
+                         .entries(options[:pages].to_i)
+    pretty_print begin
+      options[:sort]=='count' ? entries.sort_by { |ent| -ent.count } : entries
+    end
+  rescue
+    abort "something go wrong."
+  end
+
+  desc "merge *URLs", "Merge counts for same entries on several URLs"
+  option :pages, aliases:"-p", default:1
+  option :sort, aliases:"-s", default:"count"
+  def merge(*urls)
+    abort "At least 2 urls needed." if urls.size < 2
+    entries = urls.map do |url|
+      HatebuEntry.new(url, options[:sort]).entries options[:pages].to_i
+    end
+    merged = entries.inject {|mem, ent| HatebuEntry::Entry.merge(mem, ent) }
+    pretty_print merged.sort_by { |ent| -ent.count }
+  end
+  
+  desc "version", "Show HatebuEntry version"
+  def version
+    puts "HatebuEntry #{HatebuEntry::VERSION} (c) 2013 kyoendo"
+  end
+  map "-v" => :version
+
+  no_commands do
+    def pretty_print(entries)
+      lines = entries.map do |ent|
+        "%5d: %s (%s)" % [ent.count, c(ent.title), ent.link]
+      end
+      puts lines
+    end
+
+    def c(str, n='32')
+      "\e[#{n}m#{str}\e[0m"
+    end
+  end
+end

data/lib/hatebu_entry/version.rb

@@ -0,0 +1,3 @@
+class HatebuEntry
+  VERSION = "0.0.2"
+end

data/lib/hatebu_entry.rb

@@ -0,0 +1,144 @@
+require "hatebu_entry/version"
+require 'hatebu_entry/command'
+require 'open-uri'
+require 'json'
+require 'date'
+require 'cgi'
+
+require 'nokogiri'
+
+class HatebuEntry
+  # link: uri string
+  # count: integer
+  # title: string
+  class Entry < Struct.new(:link, :count, :title)
+    alias :url :link
+    alias :bookmarks :count
+    def date
+      @date ||= Date.parse(link)
+    rescue ArgumentError
+      nil
+    end
+
+    def to_s
+      "%5d: %s (%s)" % [count, title, link]
+    end
+
+    # to find same entry but other hosts
+    def homogeneous?(other)
+      return false if self == other
+      return nil if [self, other].any? { |e| e.date.nil? }
+      self.date == other.date &&
+          title_similar?(self.title, other.title)
+    end
+    alias :same? :homogeneous?
+
+    def title_similar?(a, b)
+      a, b = [a, b].map { |str| str.gsub(/\w+/, '')[0..5] }
+      a == b
+    end
+    private :title_similar?
+
+    class MergeError < StandardError; end
+
+    # Merge its count
+    def merge(other)
+      count = self.count + other.count
+      if block_given? && !yield(self, other)
+        raise MergeError, "They can't merge"
+      else
+        Entry.new(self.link, count, self.title)
+      end
+    end
+
+    # Merge two entry lists
+    def self.merge(ls_a, ls_b)
+      if [ls_a, ls_b].all? { |ls| ls.is_a? Entry }
+        raise ArgumentError, 'Arguments must be entry objects'
+      end
+      entries = []
+      ls_a.each do |a|
+        if m = ls_b.detect { |b| a.homogeneous? b }
+          entries.push a.merge(m)
+          ls_b.delete(m)
+        else
+          entries.push a
+        end
+      end
+      entries + ls_b
+    end
+  end
+
+  attr_accessor :params
+  #sort: count, eid or hot
+  def initialize(site, sort='count')
+    @params = {url: site, sort: sort, of: 0*20}
+  end
+
+  def entries(pages=0)
+    if pages <= 0
+      get_entries(:jsonp) # get 10 entries with jsonp
+    else
+      # get 20 entries per page with html
+      mutex = Mutex.new
+      pages.times.map { |i|
+        Thread.fork(i) do |_i|
+          mutex.synchronize {
+            params.update(of: _i*20)
+            get_entries(:html)
+          }
+        end
+      }.map(&:value).flatten
+    end
+  end
+
+  def get_entries(api)
+    entries = 
+      case api
+      when :jsonp then parse_jsonp(call_hatena_entry_api :jsonp)
+      when :html  then parse_html(call_hatena_entry_api :html)
+      end
+    entries.map { |h| Entry.new h['link'], Integer(h['count']), h['title'] }
+  end
+
+  def call_hatena_entry_api(api)
+    uri = {jsonp: build_uri('/json?'), html: build_uri('?')}[api]
+    get uri
+  end
+
+  def get(uri)
+    open(uri).read
+  rescue => e
+    abort "HTTP Access Error: #{e.response}"
+  end
+
+  HatenaURI = "http://b.hatena.ne.jp/entrylist"
+
+  def build_uri(joint, params=@params)
+    HatenaURI + joint + build_params(params)
+  end
+
+  def build_params(params)
+    params.map { |k, v| "#{h k}=#{h v}" } * '&'
+  end
+
+  def h(str)
+    CGI.escape(str.to_s)
+  end
+
+  def parse_jsonp(jsonp)
+    jsonp.scan(/{.+?}/).map { |data| JSON.parse data }
+  end
+
+  def parse_html(html)
+    entries = []
+    doc = Nokogiri::HTML(html)
+    doc.css('li.entry-unit').each do |ent|
+      count = ent['data-bookmark-count'].to_i
+      a = ent.at('.entry-contents a')
+      title, href = a['title'], a['href']
+      entries << {'link' => href, 'count' => count, 'title' => title}
+    end
+    entries
+  end
+end

data/sample/sample.rb

@@ -0,0 +1,29 @@
+require 'hatebu_entry'
+
+githubio = 'http://melborne.github.io'
+githubcom = 'http://melborne.github.com'
+hatena = 'http://d.hatena.ne.jp/keyesberry'
+
+def get_entries(url_list, pages=1)
+  url_list.map do |url|
+    puts "Bookmark entries retrieving from #{url}..."
+    HatebuEntry.new(url).entries(pages).tap do |s|
+      puts "#{s.size} entries retrieved."
+    end
+  end
+end
+
+gitio_ent, gitcom_ent, hatena_ent = get_entries([githubio, githubcom, hatena], 5)
+
+puts "\nMerging entries..."
+
+entries = HatebuEntry::Entry.merge(gitio_ent, gitcom_ent)
+entries = HatebuEntry::Entry.merge(entries, hatena_ent)
+
+puts "\nFollowing is top 20 entries."
+puts
+
+puts entries.lazy
+            .sort_by{ |e| -e.count }
+            .map { |h| "%i: %s (%s)" % [h.count, h.title, h.link] }
+            .take(20)

data/spec/hatebu_entry_spec.rb

@@ -0,0 +1,149 @@
+require 'spec_helper'
+
+describe HatebuEntry do
+  it 'should have a version number' do
+    HatebuEntry::VERSION.should_not be_nil
+  end
+
+  before(:each) do
+    FakeWeb.clean_registry
+    site = "http://melborne.github.com"
+    @hent = HatebuEntry.new(site)
+  end
+
+  describe '#build_json_uri' do
+    it 'returns json uri for Hatena bookmark entry' do
+      expected = "http://b.hatena.ne.jp/entrylist/json?" +
+                 "url=http%3A%2F%2Fmelborne.github.com&sort=count&of=0"
+      expect(@hent.build_uri '/json?').to eq expected
+    end
+  end
+
+  describe '#call_hatena_entry_api' do
+    it 'returns jsonp data of Hatena bookmark info' do
+      uri = @hent.build_uri('/json?')
+      mock_hatebu_entry_api(uri)
+      expect(@hent.call_hatena_entry_api :jsonp).to match /^\(\[{"link":.+}\]\);$/
+    end
+
+    it 'returns empty jsonp data with a no bookmarked site' do
+      site = "http://melborne.github.co.jp"
+      hent = HatebuEntry.new(site)
+      mock_hatebu_entry_api('http://sample.com', 'no_entry.jsonp')
+      expect(hent.call_hatena_entry_api :jsonp).to eq "([]);"
+    end
+  end
+
+  describe '#get_entries' do
+    context 'gets Bookmark entries with jsonp' do
+      it 'returns entries with an array' do
+        uri = @hent.build_uri('/json?')
+        mock_hatebu_entry_api(uri)
+        entries = @hent.get_entries(:jsonp)
+        expect(entries.size).to eq 10
+        expect(entries.first).to be_instance_of(HatebuEntry::Entry)
+        expect(entries.first.count).to eq 1377
+      end
+    end
+
+    context 'gets Bookmark entries with html' do
+      it 'returns entries with an array' do
+        uri = @hent.build_uri('?')
+        mock_hatebu_entry_api(uri, 'hatebu_entry0.html')
+        entries = @hent.get_entries(:html)
+        expect(entries.size).to eq 20
+        expect(entries.first).to be_instance_of(HatebuEntry::Entry)
+      end
+    end
+  end
+
+  describe '#parse_jsonp' do
+    it 'returns an array of hashes containing entry data' do
+      str = fixture('hatebu_entry.jsonp')
+      entries = @hent.parse_jsonp(str)
+      expect(entries).to be_instance_of(Array)
+      expect(entries.first).to be_instance_of(Hash)
+    end
+  end
+
+  describe '#parse_html' do
+    it 'returns an array of hashes containing entry data' do
+      str = fixture('hatebu_entry0.html')
+      entries = @hent.parse_html(str)
+      expect(entries).to be_instance_of(Array)
+      expect(entries.first).to be_instance_of(Hash)
+    end
+  end
+
+  describe '#entries' do
+    it 'returns entries with jsonp' do
+      uri = @hent.build_uri('/json?')
+      mock_hatebu_entry_api(uri)
+      entries = @hent.entries
+      expect(entries.size).to eq 10
+    end
+
+    context 'get entries with html' do
+      before(:each) do
+        uri = @hent.build_uri('?')
+        3.times do |i|
+          @hent.params.update(of: i*20)
+          mock_hatebu_entry_api(uri, "hatebu_entry#{i}.html")
+        end
+      end
+
+      it 'returns first 20 entries' do
+        entries = @hent.entries(1)
+        expect(entries.size).to eq 20
+      end
+
+      it 'returns first 60 entries' do
+        entries = @hent.entries(3)
+        expect(entries.size).to eq 60
+        expect(entries.last.title).to match /Graphviz/
+      end
+    end
+  end
+end
+
+describe HatebuEntry::Entry do
+  let(:ent) { HatebuEntry::Entry }
+  before(:each) do
+    @ent1 = ent.new("http://d.hatena.ne.jp/keyesberry/20130304/p1", 20,"知って得する！５５のRubyのトリビアな記法")
+    @ent2 = ent.new("http://melborne.github.io/2013/03/04/ruby/", 10,"知って得する！５５のRubyのトリビアな記法")
+    @ent3 = ent.new("http://melborne.github.com/2013/03/05/ruby/", 10,"知って得する！５５のRubyのトリビアな記法")
+    @ent4 = ent.new("http://melborne.github.com/2013/03/04/ruby/", 10,"５５のRubyのトリビアな記法")
+  end
+
+  describe '#homogeneous?' do
+    it 'returns true when date and title are same' do
+      expect(@ent1.homogeneous? @ent2).to be_true
+    end
+
+    it 'returns false when title are same but date are not' do
+      expect(@ent1.homogeneous? @ent3).to be_false
+    end
+
+    it 'returns false when date are same but title are not' do
+      expect(@ent1.homogeneous? @ent4).to be_false
+    end
+  end
+
+  describe '#merge' do
+    it 'merge others count amount to self count' do
+      merged = @ent1.merge(@ent2)
+      expect(merged.count).to eq 30
+    end
+
+    context 'merge when these are homogeneous' do
+      it 'should success when they are homogeneous' do
+        merged = @ent1.merge(@ent2) { |a, b| a.homogeneous? b }
+        expect(merged.count).to eq 30
+      end
+
+      it 'should fail when they are not homogeneous' do
+        expect{ @ent1.merge(@ent3) { |a, b| a.homogeneous? b } }.to raise_error(HatebuEntry::Entry::MergeError)
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,19 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'hatebu_entry'
+require 'fakeweb'
+
+module HelperMethods
+  def fixture(name)
+    File.read("#{__dir__}/support/#{name}")
+  end
+
+  def mock_hatebu_entry_api(uri, fix='hatebu_entry.jsonp')
+    response ||= fixture(fix)
+    FakeWeb.register_uri(:get, uri, :body => response)
+  rescue Errno::ENOENT
+    response = fixture("no_entry.jsonp")
+    retry
+  end
+end
+
+RSpec.configuration.include(HelperMethods)

data/spec/support/hatebu_entry.jsonp

@@ -0,0 +1 @@
+([{"link":"http://melborne.github.com/2012/04/09/to-newbie/","count":"1377","title":"これからRubyを始める人たちへ"},{"link":"http://melborne.github.com/2013/03/04/ruby-trivias-you-should-know-4/","count":"633","title":"知って得する！５５のRubyのトリビアな記法"},{"link":"http://melborne.github.com/2013/02/25/i-wanna-say-something-about-rubys-case/","count":"514","title":"Rubyのcaseを〇〇(言語名)のswitch文だと思っている人たちにぼ..."},{"link":"http://melborne.github.com/2011/06/22/21-Ruby-21-Trivia-Notations-you-should-know-in-Ruby/","count":"442","title":"知って得する21のRubyのトリビアな記法"},{"link":"http://melborne.github.com/2012/09/09/understand-js-oop-with-ruby-brain/","count":"435","title":"Ruby脳が理解するJavaScriptのオブジェクト指向"},{"link":"http://melborne.github.com/2012/12/25/ebooks-for-learning-ruby/","count":"269","title":"今年の冬休みに電子書籍であなたがRubyを習得しなければい..."},{"link":"http://melborne.github.com/2012/07/16/ruby-methods-analysis/","count":"236","title":"Ruby、君のオブジェクトはなんて呼び出せばいいの？"},{"link":"http://melborne.github.com/2013/01/21/why-fp-with-ruby/","count":"187","title":"Rubyを使って「なぜ関数プログラミングは重要か」を読み解..."},{"link":"http://melborne.github.com/2012/09/15/understand-js-oop-with-ruby-brain-2/","count":"178","title":"Ruby脳が理解するJavaScriptのオブジェクト指向（その２）"},{"link":"http://melborne.github.com/2013/01/24/csv-table-method-is-awesome/","count":"145","title":"Ruby標準添付ライブラリcsvのCSV.tableメソッドが最強な件につ..."}]);