qiita-export 0.0.2 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +4 -0
- data/lib/qiita-export.rb +1 -0
- data/lib/qiita-export/article.rb +7 -2
- data/lib/qiita-export/comment.rb +42 -0
- data/lib/qiita-export/config.rb +25 -8
- data/lib/qiita-export/exporter.rb +7 -2
- data/lib/qiita-export/fetcher/api_end_point.rb +23 -9
- data/lib/qiita-export/fetcher/api_fetcher.rb +0 -4
- data/lib/qiita-export/fetcher/base.rb +1 -1
- data/lib/qiita-export/fetcher/kobito_fetcher.rb +1 -1
- data/lib/qiita-export/fetcher/pagination_fetcher.rb +2 -4
- data/lib/qiita-export/fetcher/url_fetcher.rb +11 -2
- data/lib/qiita-export/version.rb +1 -1
- data/qiita-export.gemspec +2 -0
- data/spec/qiita-export/config_spec.rb +28 -0
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 07a761de535b3fac30e81ce02879ccecad548aaa
|
4
|
+
data.tar.gz: f7e388a44e4e543b30beca43b30b2d73d125999f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 55c498a11247a4179ecf9987af52a046327931449471a160bedeab72deff77c43d5ea0d93159e1ab6ed9e08989f9fdbb0283561520a08a1c05090a14e8c85076
|
7
|
+
data.tar.gz: 09558c17f085b0d3ea750728bd6ca1f807dc6adc01fe594681cbc89cdb76c2d6a6d28e54bf048867bc2298493f205a26f68761ed46fe3a7e4ee34e58951f9445
|
data/README.md
CHANGED
@@ -2,6 +2,9 @@
|
|
2
2
|
|
3
3
|
export tool for Qiita(http://qiita.com/).
|
4
4
|
|
5
|
+
qiita-export | RubyGems.org
|
6
|
+
https://rubygems.org/gems/qiita-export/
|
7
|
+
|
5
8
|
## Installation
|
6
9
|
|
7
10
|
Add this line to your Gemfile:
|
@@ -33,6 +36,7 @@ Usage: qiita-export [options]
|
|
33
36
|
-t, --team=teamname export Qiita Team articles only.
|
34
37
|
-T, --team-all export Qiita Team all articles.
|
35
38
|
-i, --image export with images.
|
39
|
+
-c, --comment export with comments.
|
36
40
|
-h, --html export in html format(experimental).
|
37
41
|
-o, --output-dir=dirpath specify the full path of destination directory.
|
38
42
|
-a, --api-token=token specify API token for Qiita.
|
data/lib/qiita-export.rb
CHANGED
data/lib/qiita-export/article.rb
CHANGED
@@ -33,15 +33,20 @@ module QiitaExport
|
|
33
33
|
end
|
34
34
|
|
35
35
|
def save
|
36
|
-
save_dir = File.join(Config.export_dir_path
|
36
|
+
save_dir = File.join(Config.export_dir_path(@url), @key)
|
37
37
|
|
38
38
|
FileUtils.makedirs(save_dir) unless Dir.exists?(save_dir)
|
39
39
|
|
40
40
|
file_path = File.join(save_dir, Config.filename(title))
|
41
41
|
File.open(file_path, "w") { |f| f.write export_content }
|
42
|
-
|
42
|
+
|
43
|
+
if Config.image_export?
|
43
44
|
@images.each { |image| image.save(save_dir) }
|
44
45
|
end
|
46
|
+
|
47
|
+
if Config.comment_export?
|
48
|
+
Comment.new(@url).save(save_dir)
|
49
|
+
end
|
45
50
|
end
|
46
51
|
|
47
52
|
private
|
@@ -0,0 +1,42 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
require 'fileutils'
|
3
|
+
require 'uri'
|
4
|
+
|
5
|
+
module QiitaExport
|
6
|
+
class Comment
|
7
|
+
attr_reader :key
|
8
|
+
|
9
|
+
def initialize(url)
|
10
|
+
@endpoint = Fetcher::ApiEndPoint.instance(:comment)
|
11
|
+
@url = url
|
12
|
+
end
|
13
|
+
|
14
|
+
def request_header
|
15
|
+
Config.has_api_token? ? Config.auth_header : Config.default_header
|
16
|
+
end
|
17
|
+
|
18
|
+
def filename
|
19
|
+
"comments.json"
|
20
|
+
end
|
21
|
+
|
22
|
+
def save(path)
|
23
|
+
comments = find_comments
|
24
|
+
return if comments =~ /^\[\]$/i
|
25
|
+
|
26
|
+
file_path = File.join(File.expand_path(path), filename)
|
27
|
+
open(file_path, 'wb') do |out|
|
28
|
+
out.write(comments)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def find_comments
|
33
|
+
url = @endpoint.url(article_url: @url)
|
34
|
+
open(url, request_header) do |io|
|
35
|
+
io.read
|
36
|
+
end
|
37
|
+
rescue => e
|
38
|
+
$stderr.puts "#{e} : #{url}"
|
39
|
+
raise
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
data/lib/qiita-export/config.rb
CHANGED
@@ -27,6 +27,7 @@ module QiitaExport
|
|
27
27
|
opt.on('-t', '--team=teamname', 'export Qiita Team articles only.') { |v| @option[:team] = v }
|
28
28
|
opt.on('-T', '--team-all', 'export Qiita Team all articles.') { |v| @option[:'team-all'] = v }
|
29
29
|
opt.on('-i', '--image', 'export with images.') { |v| @option[:image] = v }
|
30
|
+
opt.on('-c', '--comment', 'export with comments.') { |v| @option[:comment] = v }
|
30
31
|
opt.on('-h', '--html', 'export in html format(experimental).') { |v| @option[:html] = v }
|
31
32
|
opt.on('-o', '--output-dir=dirpath', 'specify the full path of destination directory.') { |v| @option[:'output-dir'] = v }
|
32
33
|
opt.on('-a', '--api-token=token', 'specify API token for Qiita.') { |v| @option[:'api-token'] = v }
|
@@ -111,6 +112,10 @@ module QiitaExport
|
|
111
112
|
@option[:image]
|
112
113
|
end
|
113
114
|
|
115
|
+
def comment_export?
|
116
|
+
@option[:comment]
|
117
|
+
end
|
118
|
+
|
114
119
|
def team_url?(url)
|
115
120
|
url !~ /^https?:\/\/qiita\.com/
|
116
121
|
end
|
@@ -126,6 +131,19 @@ module QiitaExport
|
|
126
131
|
end
|
127
132
|
end
|
128
133
|
|
134
|
+
USER_ID_PATTERN = Regexp.new("https?://[^/?&#]+/([^/]+)/")
|
135
|
+
def user_id(url = nil)
|
136
|
+
if url.nil?
|
137
|
+
@option[:'user-id']
|
138
|
+
else
|
139
|
+
url.match(USER_ID_PATTERN)[1]
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
def article_key(url)
|
144
|
+
File.basename(url)
|
145
|
+
end
|
146
|
+
|
129
147
|
def article_urls
|
130
148
|
urls = []
|
131
149
|
urls << @option[:url] if present?(@option[:url])
|
@@ -180,16 +198,15 @@ module QiitaExport
|
|
180
198
|
header
|
181
199
|
end
|
182
200
|
|
183
|
-
def
|
184
|
-
|
201
|
+
def output_dir
|
202
|
+
return "" unless file_export?
|
203
|
+
File.expand_path(@option[:'output-dir'].strip)
|
185
204
|
end
|
186
205
|
|
187
|
-
def export_dir_path
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
File.expand_path(@option[:'output-dir'].strip)
|
192
|
-
end
|
206
|
+
def export_dir_path(url)
|
207
|
+
team = team_name(url)
|
208
|
+
user = user_id(url)
|
209
|
+
File.join(output_dir, team, user)
|
193
210
|
end
|
194
211
|
|
195
212
|
def filename(title)
|
@@ -37,8 +37,13 @@ module QiitaExport
|
|
37
37
|
end
|
38
38
|
|
39
39
|
def export_file(articles)
|
40
|
-
|
41
|
-
|
40
|
+
if articles.length < 1
|
41
|
+
$stdout.puts("no articles found.")
|
42
|
+
return
|
43
|
+
end
|
44
|
+
|
45
|
+
$stdout.puts "export articles to #{Config.output_dir}"
|
46
|
+
|
42
47
|
articles.each do |article|
|
43
48
|
article.save
|
44
49
|
end
|
@@ -3,16 +3,22 @@
|
|
3
3
|
module QiitaExport::Fetcher
|
4
4
|
class ApiEndPoint
|
5
5
|
|
6
|
+
MAX_PER_PAGE = 100
|
7
|
+
|
6
8
|
def initialize
|
7
9
|
@config = ::QiitaExport::Config
|
8
10
|
end
|
9
11
|
|
10
|
-
def
|
12
|
+
def url(article_url: nil, page: 1)
|
11
13
|
raise NotImplementedError.new("You must implement #{self.class}##{__method__}")
|
12
14
|
end
|
13
15
|
|
14
16
|
def self.instance(end_point_sym)
|
15
17
|
case end_point_sym
|
18
|
+
when :item
|
19
|
+
ItemEndPoint.new
|
20
|
+
when :comment
|
21
|
+
CommentEndPoint.new
|
16
22
|
when :user
|
17
23
|
UserEndPoint.new
|
18
24
|
when :team
|
@@ -23,19 +29,27 @@ module QiitaExport::Fetcher
|
|
23
29
|
end
|
24
30
|
end
|
25
31
|
|
26
|
-
class
|
27
|
-
|
32
|
+
class ItemEndPoint < ApiEndPoint
|
33
|
+
def url(article_url: nil, page: 1)
|
34
|
+
"https://#{@config.api_domain(article_url)}/api/v2/items/#{@config.article_key(article_url)}"
|
35
|
+
end
|
36
|
+
end
|
28
37
|
|
29
|
-
|
30
|
-
|
38
|
+
class CommentEndPoint < ApiEndPoint
|
39
|
+
def url(article_url: nil, page: 1)
|
40
|
+
"https://#{@config.api_domain(article_url)}/api/v2/items/#{@config.article_key(article_url)}/comments"
|
31
41
|
end
|
32
42
|
end
|
33
43
|
|
34
|
-
class
|
35
|
-
|
44
|
+
class UserEndPoint < ApiEndPoint
|
45
|
+
def url(article_url: nil, page: 1)
|
46
|
+
"https://#{@config.api_domain}/api/v2/users/#{@config.user_id}/items?page=#{page}&per_page=#{MAX_PER_PAGE}"
|
47
|
+
end
|
48
|
+
end
|
36
49
|
|
37
|
-
|
38
|
-
|
50
|
+
class TeamEndPoint < ApiEndPoint
|
51
|
+
def url(article_url: nil, page: 1)
|
52
|
+
"https://#{@config.api_domain}/api/v2/items?page=#{page}&per_page=#{MAX_PER_PAGE}"
|
39
53
|
end
|
40
54
|
end
|
41
55
|
end
|
@@ -20,10 +20,6 @@ module QiitaExport::Fetcher
|
|
20
20
|
::QiitaExport::Article.new(key, url, title, raw_body, rendered_body, created_at, updated_at, user_id)
|
21
21
|
end
|
22
22
|
|
23
|
-
def article_key(url)
|
24
|
-
File.basename(url)
|
25
|
-
end
|
26
|
-
|
27
23
|
def request_header
|
28
24
|
has_api_token? ? auth_header : default_header
|
29
25
|
end
|
@@ -5,8 +5,6 @@ require 'open-uri'
|
|
5
5
|
module QiitaExport::Fetcher
|
6
6
|
class PaginationFetcher < ApiFetcher
|
7
7
|
|
8
|
-
PER_PAGE = 100
|
9
|
-
|
10
8
|
def initialize(endpoint_sym)
|
11
9
|
super()
|
12
10
|
@endpoint = ApiEndPoint.instance(endpoint_sym)
|
@@ -33,8 +31,8 @@ module QiitaExport::Fetcher
|
|
33
31
|
private
|
34
32
|
|
35
33
|
def paginate_articles(page)
|
36
|
-
url = @endpoint.
|
37
|
-
open(
|
34
|
+
url = @endpoint.url(page: page)
|
35
|
+
open(url, request_header) do |io|
|
38
36
|
JSON.parse(io.read)
|
39
37
|
end
|
40
38
|
rescue => e
|
@@ -5,6 +5,11 @@ require 'open-uri'
|
|
5
5
|
module QiitaExport::Fetcher
|
6
6
|
class UrlFetcher < ApiFetcher
|
7
7
|
|
8
|
+
def initialize
|
9
|
+
super()
|
10
|
+
@endpoint = ApiEndPoint.instance(:item)
|
11
|
+
end
|
12
|
+
|
8
13
|
def find_articles
|
9
14
|
articles = []
|
10
15
|
article_urls.each do |url|
|
@@ -16,10 +21,14 @@ module QiitaExport::Fetcher
|
|
16
21
|
|
17
22
|
private
|
18
23
|
|
19
|
-
def find_article(
|
20
|
-
|
24
|
+
def find_article(article_url)
|
25
|
+
url = @endpoint.url(article_url: article_url)
|
26
|
+
open(url, request_header) do |io|
|
21
27
|
JSON.parse(io.read)
|
22
28
|
end
|
29
|
+
rescue => e
|
30
|
+
$stderr.puts "#{e} : #{url}"
|
31
|
+
raise
|
23
32
|
end
|
24
33
|
end
|
25
34
|
end
|
data/lib/qiita-export/version.rb
CHANGED
data/qiita-export.gemspec
CHANGED
@@ -5,6 +5,7 @@ require 'qiita-export'
|
|
5
5
|
|
6
6
|
describe QiitaExport::Config do
|
7
7
|
let(:valid_url) { 'http://qiita.com/akishin/items/61630d628f4c8e141ef2' }
|
8
|
+
let(:valid_team_url) { 'http://example.qiita.com/foo@github/items/99999999999999999999' }
|
8
9
|
let(:valid_kobito_db) { File.expand_path(QiitaExport::Config::DEFAULT_KOBITO_DB) }
|
9
10
|
let(:valid_team_name) { 'example' }
|
10
11
|
let(:valid_argv) {
|
@@ -42,6 +43,33 @@ describe QiitaExport::Config do
|
|
42
43
|
|
43
44
|
end
|
44
45
|
|
46
|
+
describe ".user_id" do
|
47
|
+
context "with user_id option" do
|
48
|
+
subject do
|
49
|
+
described_class.configure(['--user-id', 'foo'])
|
50
|
+
end
|
51
|
+
it { expect(subject.user_id).to eq('foo') }
|
52
|
+
end
|
53
|
+
|
54
|
+
context "without user_id option" do
|
55
|
+
subject do
|
56
|
+
described_class.configure(valid_argv)
|
57
|
+
end
|
58
|
+
it { expect(subject.user_id).to be_nil }
|
59
|
+
it { expect(subject.user_id(valid_url)).to eq('akishin') }
|
60
|
+
it { expect(subject.user_id(valid_team_url)).to eq('foo@github') }
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
describe ".export_dir_path" do
|
65
|
+
subject do
|
66
|
+
described_class.configure(['--output-dir', '/tmp/output'])
|
67
|
+
end
|
68
|
+
|
69
|
+
it { expect(subject.export_dir_path(valid_url)).to eq('/tmp/output/qiita/akishin') }
|
70
|
+
it { expect(subject.export_dir_path(valid_team_url)).to eq('/tmp/output/example/foo@github') }
|
71
|
+
end
|
72
|
+
|
45
73
|
end
|
46
74
|
|
47
75
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: qiita-export
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shin Akiyama
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-04-
|
11
|
+
date: 2016-04-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sqlite3
|
@@ -82,6 +82,7 @@ files:
|
|
82
82
|
- bin/qiita-export
|
83
83
|
- lib/qiita-export.rb
|
84
84
|
- lib/qiita-export/article.rb
|
85
|
+
- lib/qiita-export/comment.rb
|
85
86
|
- lib/qiita-export/config.rb
|
86
87
|
- lib/qiita-export/exporter.rb
|
87
88
|
- lib/qiita-export/fetcher/api_end_point.rb
|
@@ -108,7 +109,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
108
109
|
requirements:
|
109
110
|
- - ">="
|
110
111
|
- !ruby/object:Gem::Version
|
111
|
-
version:
|
112
|
+
version: 2.0.0
|
112
113
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
113
114
|
requirements:
|
114
115
|
- - ">="
|