qiita-export 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6442dd02c9dd12b8a17344ad0a9e3f6dd780c371
-  data.tar.gz: 1463061332a477bf955edcbd435db8a3c0fb6279
+  metadata.gz: 821a3351e6e0d5ae3a620e12828d645e9b985678
+  data.tar.gz: 18d4e090f2d8f10f06e81344d9f13772cdace11c
 SHA512:
-  metadata.gz: 4668ef573aaa449d7d37248bc060a05700f555fd3bf16158d412b2809f20794130c1e5955d8cf15805966e2801ee861595a56deb79ae0970892007d4b76b943b
-  data.tar.gz: ec2cb0016242ce4a2ee41e0d8926facb234e4f77b06142a4e73371ff3f950f97f6f92995529f25c8618f1b342d14fdde3c6a315e899101b22b80e90e22ecacbd
+  metadata.gz: 1551e1b916754310037f0993f8ed1048bd27fdd11f7eb5b57e209e68799ea30e5a509d69846183eae8ea7105703309814a9c869742ce138013ef24245addd994
+  data.tar.gz: acf2bea60094739fdc10959eaf703ddcf27381fd587de0db4900de29482b008a169188fdf9415908c2e8fe1646d2b55dba358145efd5a3c586fe5b0e72bff865

data/README.md

@@ -4,10 +4,10 @@ export tool for Qiita(http://qiita.com/).
 
 ## Installation
 
-Add this line to your application's Gemfile:
+Add this line to your Gemfile:
 
 ```ruby
-gem 'qiita_export'
+gem 'qiita-export'
 ```
 
 And then execute:
@@ -19,7 +19,7 @@ $ bundle
 Or install it yourself as:
 
 ```
-$ gem install qiita_export
+$ gem install qiita-export
 ```
 
 ## Usage
@@ -31,10 +31,12 @@ Usage: qiita-export [options]
     -U, --user-id=user_id            specify the userid for the Qiita.
     -k, --kobito=[Kobito.db]         export Kobito.app database.
     -t, --team=teamname              export Qiita Team articles only.
+    -T, --team-all                   export Qiita Team all articles.
     -i, --image                      export with images.
     -h, --html                       export in html format(experimental).
     -o, --output-dir=dirpath         specify the full path of destination directory.
     -a, --api-token=token            specify API token for Qiita.
+    -e, --exclude-pattern=regexp     specify the regexp pattern to exclude article title.
 ```
 
 ## Contributing

data/lib/qiita-export.rb

@@ -3,8 +3,9 @@ require 'qiita-export/image'
 require 'qiita-export/article'
 require 'qiita-export/config'
 require 'qiita-export/exporter'
+require 'qiita-export/fetcher/api_end_point'
 require 'qiita-export/fetcher/base'
 require 'qiita-export/fetcher/kobito_fetcher'
 require 'qiita-export/fetcher/api_fetcher'
 require 'qiita-export/fetcher/url_fetcher'
-require 'qiita-export/fetcher/user_fetcher'
+require 'qiita-export/fetcher/pagination_fetcher'

data/lib/qiita-export/config.rb

@@ -6,6 +6,7 @@ module QiitaExport
     HOME_CONFIG_FILE = "~/.qiita-exportrc"
     LOCAL_CONFIG_FILE = "./.qiita-exportrc"
 
+    DEFAULT_HOST = "qiita.com"
     DEFAULT_USER_AGENT = "QiitaExport Gem #{QiitaExport::VERSION}"
 
     DEFAULT_HEADER = {
@@ -19,15 +20,17 @@ module QiitaExport
         @option = {}
         @parser = OptionParser.new do |opt|
           opt.version = QiitaExport::VERSION
-          opt.on('-u', '--url=url',            'specify the URL for the Qiita(or Qiita Team).')   { |v| @option[:url] = v }
-          opt.on('-l', '--url-list=filepath',  'specify the file path of the URL list.')          { |v| @option[:'url-list'] = v }
-          opt.on('-U', '--user-id=user_id',    'specify the userid for the Qiita.')               { |v| @option[:'user-id'] = v }
-          opt.on('-k', '--kobito=[Kobito.db]', 'export Kobito.app database.')                     { |v| @option[:kobito] = v }
-          opt.on('-t', '--team=teamname',      'export Qiita Team articles only.')                { |v| @option[:team] = v }
-          opt.on('-i', '--image',              'export with images.')                             { |v| @option[:image] = v }
-          opt.on('-h', '--html',               'export in html format(experimental).')            { |v| @option[:html] = v }
-          opt.on('-o', '--output-dir=dirpath', 'specify the full path of destination directory.') { |v| @option[:'output-dir'] = v }
-          opt.on('-a', '--api-token=token',    'specify API token for Qiita.')                    { |v| @option[:'api-token'] = v }
+          opt.on('-u', '--url=url',                'specify the URL for the Qiita(or Qiita Team).')        { |v| @option[:url] = v }
+          opt.on('-l', '--url-list=filepath',      'specify the file path of the URL list.')               { |v| @option[:'url-list'] = v }
+          opt.on('-U', '--user-id=user_id',        'specify the userid for the Qiita.')                    { |v| @option[:'user-id'] = v }
+          opt.on('-k', '--kobito=[Kobito.db]',     'export Kobito.app database.')                          { |v| @option[:kobito] = v }
+          opt.on('-t', '--team=teamname',          'export Qiita Team articles only.')                     { |v| @option[:team] = v }
+          opt.on('-T', '--team-all',               'export Qiita Team all articles.')                      { |v| @option[:'team-all'] = v }
+          opt.on('-i', '--image',                  'export with images.')                                  { |v| @option[:image] = v }
+          opt.on('-h', '--html',                   'export in html format(experimental).')                 { |v| @option[:html] = v }
+          opt.on('-o', '--output-dir=dirpath',     'specify the full path of destination directory.')      { |v| @option[:'output-dir'] = v }
+          opt.on('-a', '--api-token=token',        'specify API token for Qiita.')                         { |v| @option[:'api-token'] = v }
+          opt.on('-e', '--exclude-pattern=regexp', 'specify the regexp pattern to exclude article title.') { |v| @option[:'exclude-pattern'] = v }
         end
 
         # load home config
@@ -62,6 +65,18 @@ module QiitaExport
         if kobito? && api?
           fail ArgumentError.new("if you specify option --kobito, you cannot specify option --url, --url-list and --user-id.")
         end
+
+        if !kobito? && !api?
+          fail ArgumentError.new("one of the required options --kobito, --url, --url-list and --user-id must be specified.")
+        end
+
+        if !kobito? && team? && !has_api_token?
+          fail ArgumentError.new("if you specify option --team, option --api-token is required.")
+        end
+
+        if team_all? && !team?
+          fail ArgumentError.new("if you specify option --team-all, option --team is required.")
+        end
       end
 
       def kobito?
@@ -69,7 +84,7 @@ module QiitaExport
       end
 
       def api?
-        present?(@option[:url]) || present?(@option[:'url-list']) || present?(@option[:'user-id'])
+        present?(@option[:url]) || present?(@option[:'url-list']) || present?(@option[:'user-id']) || team_all?
       end
 
       def user?
@@ -80,6 +95,10 @@ module QiitaExport
         present?(@option[:team])
       end
 
+      def team_all?
+        @option.key?(:'team-all')
+      end
+
       def file_export?
         present?(@option[:'output-dir'])
       end
@@ -98,10 +117,10 @@ module QiitaExport
 
       DOMAIN_PATTERN = Regexp.new("https?://([^/]+)/")
       def team_name(url = nil)
-        if (kobito? || user?) && team?
-          @option[:team]
-        elsif api? && team_url?(url)
+        if present?(url)
           url.match(DOMAIN_PATTERN)[1].split('.')[0]
+        elsif team?
+          @option[:team]
         else
           ""
         end
@@ -131,10 +150,22 @@ module QiitaExport
         end
       end
 
+      def api_domain(url = nil)
+        if url.nil?
+          team? ? "#{team_name}.#{DEFAULT_HOST}" : DEFAULT_HOST
+        else
+          URI.parse(url).host
+        end
+      end
+
       def api_token
         @option[:'api-token']
       end
 
+      def exclude_pattern
+        @option[:'exclude-pattern']
+      end
+
       def has_api_token?
         present?(@option[:'api-token'])
       end

data/lib/qiita-export/exporter.rb

@@ -22,7 +22,9 @@ module QiitaExport
       if Config.kobito?
         QiitaExport::Fetcher::KobitoFetcher.new
       elsif Config.user?
-        QiitaExport::Fetcher::UserFetcher.new
+        QiitaExport::Fetcher::PaginationFetcher.new(:user)
+      elsif Config.team_all?
+        QiitaExport::Fetcher::PaginationFetcher.new(:team)
       else
         QiitaExport::Fetcher::UrlFetcher.new
       end

data/lib/qiita-export/fetcher/api_end_point.rb

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+
+module QiitaExport::Fetcher
+  class ApiEndPoint
+
+    def initialize
+      @config = ::QiitaExport::Config
+    end
+
+    def next_page(page)
+      raise NotImplementedError.new("You must implement #{self.class}##{__method__}")
+    end
+
+    def self.instance(end_point_sym)
+      case end_point_sym
+      when :user
+        UserEndPoint.new
+      when :team
+        TeamEndPoint.new
+      else
+        raise ArgumentError("unsupported endpont: #{end_point_sym}")
+      end
+    end
+  end
+
+  class UserEndPoint < ApiEndPoint
+    PER_PAGE = 100
+
+    def next_page(page)
+      "https://#{@config.api_domain}/api/v2/users/#{@config.user_id}/items?page=#{page}&per_page=#{PER_PAGE}"
+    end
+  end
+
+  class TeamEndPoint < ApiEndPoint
+    PER_PAGE = 100
+
+    def next_page(page)
+      "https://#{@config.api_domain}/api/v2/items?page=#{page}&per_page=#{PER_PAGE}"
+    end
+  end
+end

data/lib/qiita-export/fetcher/api_fetcher.rb

@@ -5,8 +5,6 @@ require 'uri'
 module QiitaExport::Fetcher
   class ApiFetcher < Base
 
-    DEFAULT_HOST = "qiita.com"
-
     protected
 
     def to_article(res)
@@ -30,13 +28,5 @@ module QiitaExport::Fetcher
       has_api_token? ? auth_header : default_header
     end
 
-    def api_domain(url = nil)
-      if url.nil?
-        team? ? "#{team_name}.#{DEFAULT_HOST}" : DEFAULT_HOST
-      else
-        URI.parse(url).host
-      end
-    end
-
   end
 end

data/lib/qiita-export/fetcher/base.rb

@@ -8,7 +8,8 @@ module QiitaExport::Fetcher
     def_delegators :@config, :kobito?, :kobito_db, :file_export?,
                              :export_dir_path, :team?, :team_name,
                              :article_urls, :has_api_token?, :api_token,
-                             :user_id, :default_header, :auth_header
+                             :user_id, :default_header, :auth_header,
+                             :api_domain
 
     def initialize
       @config = ::QiitaExport::Config
@@ -18,5 +19,12 @@ module QiitaExport::Fetcher
       raise NotImplementedError.new("You must implement #{self.class}##{__method__}")
     end
 
+    def exclude?(title)
+      if !@config.exclude_pattern.nil? && /#{@config.exclude_pattern}/ =~ title
+        return true
+      end
+      false
+    end
+
   end
 end

data/lib/qiita-export/fetcher/kobito_fetcher.rb

@@ -42,7 +42,8 @@ module QiitaExport::Fetcher
 
       articles = []
       while(row = rs.next)
-        articles << to_article(row)
+        article =  to_article(row)
+        articles << article unless exclude?(article.title)
       end
       articles
     ensure

data/lib/qiita-export/fetcher/pagination_fetcher.rb

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+require 'json'
+require 'open-uri'
+
+module QiitaExport::Fetcher
+  class PaginationFetcher < ApiFetcher
+
+    PER_PAGE = 100
+
+    def initialize(endpoint_sym)
+      super()
+      @endpoint = ApiEndPoint.instance(endpoint_sym)
+    end
+
+    def find_articles
+      articles = []
+      page = 1
+      while true
+        api_responses = paginate_articles(page)
+        break if api_responses.empty?
+
+        api_responses.each do |api_response|
+          article = to_article(api_response)
+          articles << article unless exclude?(article.title)
+        end
+
+        page += 1
+        sleep(0.3)
+      end
+      articles.sort { |a, b| a.created_at <=> b.created_at }
+    end
+
+    private
+
+    def paginate_articles(page)
+      url = @endpoint.next_page(page)
+      open(@endpoint.next_page(page), request_header) do |io|
+        JSON.parse(io.read)
+      end
+    rescue => e
+      $stderr.puts "#{e} : #{url}"
+      raise
+    end
+  end
+end

data/lib/qiita-export/fetcher/url_fetcher.rb

@@ -8,7 +8,8 @@ module QiitaExport::Fetcher
     def find_articles
       articles = []
       article_urls.each do |url|
-        articles << to_article(find_article(url))
+        article = to_article(find_article(url))
+        articles << article unless exclude?(article.title)
       end
       articles
     end

data/lib/qiita-export/version.rb

@@ -1,4 +1,4 @@
 module QiitaExport
-  VERSION = '0.0.1'
+  VERSION = '0.0.2'
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: qiita-export
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Shin Akiyama
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-19 00:00:00.000000000 Z
+date: 2016-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sqlite3
@@ -84,11 +84,12 @@ files:
 - lib/qiita-export/article.rb
 - lib/qiita-export/config.rb
 - lib/qiita-export/exporter.rb
+- lib/qiita-export/fetcher/api_end_point.rb
 - lib/qiita-export/fetcher/api_fetcher.rb
 - lib/qiita-export/fetcher/base.rb
 - lib/qiita-export/fetcher/kobito_fetcher.rb
+- lib/qiita-export/fetcher/pagination_fetcher.rb
 - lib/qiita-export/fetcher/url_fetcher.rb
-- lib/qiita-export/fetcher/user_fetcher.rb
 - lib/qiita-export/image.rb
 - lib/qiita-export/version.rb
 - qiita-export.gemspec

data/lib/qiita-export/fetcher/user_fetcher.rb

@@ -1,35 +0,0 @@
-# -*- coding: utf-8 -*-
-require 'json'
-require 'open-uri'
-
-module QiitaExport::Fetcher
-  class UserFetcher < ApiFetcher
-
-    PER_PAGE = 100
-
-    def find_articles
-      articles = []
-      page = 1
-      while true
-        user_articles = find_user_articles(page)
-        break if user_articles.empty?
-
-        user_articles.each do |user_article|
-          articles << to_article(user_article)
-        end
-
-        page += 1
-        sleep(0.3)
-      end
-      articles.sort { |a, b| a.created_at <=> b.created_at }
-    end
-
-    private
-
-    def find_user_articles(page)
-      open("https://#{api_domain}/api/v2/users/#{user_id}/items?page=#{page}&per_page=#{PER_PAGE}", request_header) do |io|
-        JSON.parse(io.read)
-      end
-    end
-  end
-end