komic-cli 0.1.2 → 0.1.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 8bf50bdb640316bafada1967d00f80fb0d3faab3
4
- data.tar.gz: acbd5ed6910379f34b94ae3bec6552c4046a4365
3
+ metadata.gz: 272c4b6b05cf4ad6db1a421abf853bb144065c6b
4
+ data.tar.gz: 86a88c08620850b1fb30d1b7c4db5a96e6edbba4
5
5
  SHA512:
6
- metadata.gz: 1ea45d1059dd60751a6a280bceead51aff6c32faf12eeedbbe470915b5294813e8990b8a3caa33815d79f3300eb220995de19f4035f2f268c6dac414cedf0b23
7
- data.tar.gz: fde516f1ad7c49bf457953469d3043f730d02f9429c44bd8b6faf56dc394c1efa14a473365dc0f0dc3d4721aac5e273f7b7543959a509b22267b84ad9520fcc0
6
+ metadata.gz: 35701b86c150c6ac3f1387bbbf9bee381562208160adfc4749938ed50858a958dce199888b4007a0a95d1b58dcf8a320d3221bc04d17c30ff9cfa4095d063d36
7
+ data.tar.gz: 2a9bf66cb3e0fb1e0a292a0c972f4ac8976b4197757fb3e8dd998a265d721e1a7d2598f00a121caf0a369d1dc98f9e01d7c60f8cdb572ed82e3032d18d9f2dce
data/komic-cli.gemspec CHANGED
@@ -27,6 +27,7 @@ Gem::Specification.new do |gem|
27
27
  gem.add_runtime_dependency 'jbuilder', '~> 2.3'
28
28
  gem.add_runtime_dependency 'json-schema', '~> 2.5'
29
29
  gem.add_runtime_dependency 'ruby-progressbar', '~> 1.7'
30
+ gem.add_runtime_dependency 'rubyzip', '~> 1.1'
30
31
 
31
32
  gem.add_development_dependency 'bundler', '~> 1.0'
32
33
  gem.add_development_dependency 'rake', '~> 0.8'
@@ -0,0 +1,108 @@
1
+ require 'mechanize'
2
+ require 'fileutils'
3
+ require 'mime/types'
4
+ require 'tmpdir'
5
+ require 'komic/utils'
6
+
7
+ module Komic::Builder
8
+ class DoubanAlbum
9
+ attr_reader :images
10
+
11
+ def initialize(type_string, options)
12
+ @options = options
13
+ @url = type_string
14
+ end
15
+
16
+ def images
17
+ crawler = Crawler.new
18
+ title, images = crawler.get_crawled_result(@url)
19
+ images.map do |image_path|
20
+ image = MiniMagick::Image.open(image_path)
21
+ { src: image_path, width: image.width, height: image.height }
22
+ end
23
+ end
24
+ end
25
+
26
+ class DoubanAlbum::Crawler
27
+ def initialize
28
+ @mechanize = Mechanize.new
29
+ @file_index = 0
30
+ @tmpdir = Dir.mktmpdir
31
+ @willbe_downloaded = []
32
+ end
33
+
34
+ def get_crawled_result(album_home_url)
35
+ next_link_url = album_home_url
36
+ next_link = nil
37
+ album_title = nil
38
+
39
+ @mechanize.get(album_home_url) do |page|
40
+ album_title = page.at('title').text().strip!
41
+ end
42
+
43
+ loop do
44
+ begin
45
+ crawl_album_page(next_link_url)
46
+ rescue Exception => e
47
+ puts e.message
48
+ puts e.backtrace.inspect
49
+ end
50
+
51
+ @mechanize.get(next_link_url) do |page|
52
+ next_link = page.at('link[rel="next"]')
53
+ end
54
+ break if next_link.nil?
55
+ next_link_url = next_link["href"]
56
+ end
57
+
58
+ bar = Komic::Utils.create_progress("Download images from douban", \
59
+ @willbe_downloaded.size)
60
+
61
+ image_pathes = @willbe_downloaded.map do |url|
62
+ image_path = download_image url
63
+ bar.increment
64
+ image_path
65
+ end
66
+
67
+ return album_title, image_pathes
68
+ end
69
+
70
+ def crawl_album_page(album_page_url)
71
+ @mechanize.get(album_page_url) do |page|
72
+ page.search('.photolst_photo').each do |link|
73
+ crawl_photo_page(link['href'])
74
+ end
75
+ end
76
+ end
77
+
78
+ def crawl_photo_page(photo_page_url)
79
+ link_to_large = nil
80
+ thumb_photo_url = nil
81
+
82
+ @mechanize.get(photo_page_url) do |page|
83
+ link_to_large = page.at('a[title="查看原图"]')
84
+ thumb_photo_url = page.at('.image-show-inner img')["src"]
85
+ end
86
+
87
+ unless link_to_large.nil?
88
+ @mechanize.get(link_to_large['href']) do |page|
89
+ @willbe_downloaded.push(page.at('#pic-viewer img')["src"])
90
+ end
91
+ else
92
+ @willbe_downloaded.push(thumb_photo_url)
93
+ end
94
+ end
95
+
96
+ def download_image(photo_url)
97
+ resource = @mechanize.get(photo_url)
98
+ content_type = resource["content-type"]
99
+ mime_type = MIME::Types[resource["content-type"]].first
100
+ image_path = File.expand_path( \
101
+ [@file_index, mime_type.extensions.first].join('.'), @tmpdir)
102
+ resource.save(image_path)
103
+ @file_index = @file_index + 1
104
+ sleep 2
105
+ image_path
106
+ end
107
+ end
108
+ end
@@ -0,0 +1,29 @@
1
+ require 'tempfile'
2
+ require 'komic/utils'
3
+
4
+ module Komic::Builder
5
+ class PDF
6
+ attr_reader :images
7
+
8
+ def initialize(type_string, options)
9
+ @options = options
10
+ @pdf_path = File.join(Dir.pwd, type_string)
11
+ end
12
+
13
+ def images
14
+ pdf = MiniMagick::Image.new(@pdf_path)
15
+
16
+ bar = Komic::Utils.create_progress("Extract images from pdf", pdf.pages.size)
17
+
18
+ pdf.pages.each_with_index.map do |page, idx|
19
+ will_be_write = Tempfile.new("#{idx}").path
20
+ page.write will_be_write
21
+ image = MiniMagick::Image.open(will_be_write)
22
+ image.format('jpg')
23
+ image.write Tempfile.new(["#{idx}", '.jpg']).path
24
+ bar.increment
25
+ { width: image.width, height: image.height, src: image.path }
26
+ end
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,32 @@
1
+ require 'komic/builder/pdf'
2
+ require 'komic/builder/douban_album'
3
+ require 'uri'
4
+
5
+ module Komic::Builder
6
+ class Factory
7
+ class << self
8
+ def detect_type(string)
9
+ path = File.join(Dir.pwd, string)
10
+ r_douban_album = Regexp.new "www.douban.com/photos/album/"
11
+
12
+ if string =~ URI::regexp and string =~ r_douban_album
13
+ return 'douban_album'
14
+ end
15
+
16
+ if File.exists?(path)
17
+ if File.extname(path) == '.pdf'
18
+ return 'pdf'
19
+ end
20
+ end
21
+ raise "Builder can't be found."
22
+ end
23
+
24
+ def get_builder(type_string, options)
25
+ case detect_type(type_string)
26
+ when 'pdf' then PDF.new(type_string, options)
27
+ when 'douban_album' then DoubanAlbum.new(type_string, options)
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
data/lib/komic/cli.rb CHANGED
@@ -1,25 +1,16 @@
1
1
  require 'thor'
2
- require 'komic/crawler/douban'
3
2
  require 'komic/generator/generator'
3
+ require 'komic/builder'
4
4
  require 'mini_magick'
5
5
 
6
6
  module Komic
7
7
  # This module handles the Komic executables .
8
8
  class Cli < Thor
9
9
 
10
- map '-d' => :download
11
-
12
10
  desc "download URL", "从 url 下载画册数据 (* 目前只支持豆瓣相册)"
13
11
  option :name, default: "crawled_from_douban", desc: "设定文件夹名"
14
12
  def download(url)
15
- crawler = Komic::Crawler::Douban.new
16
- title, images = crawler.get_crawled_result(url)
17
- images = images.map do |image_path|
18
- image = MiniMagick::Image.open(image_path)
19
- { src: image_path, width: image.width, height: image.height }
20
- end
21
- generator = Komic::Generator.new
22
- generator.create_package({ images: images, meta: { name: title } }, options)
13
+ dev(url)
23
14
  end
24
15
 
25
16
  desc "version", "显示版本"
@@ -38,5 +29,21 @@ module Komic
38
29
  mocks = generator.generate_mocks options
39
30
  generator.create_package({ images: mocks }, options)
40
31
  end
32
+
33
+ desc "dev", '生成 dev 的数据'
34
+ option :name, default: "dev", desc: "设定文件夹名"
35
+ def dev(type_string)
36
+ generator = Komic::Generator.new
37
+ pdf_builder = Builder::Factory.get_builder(type_string, options)
38
+ generator.create_package({ images: pdf_builder.images }, options)
39
+ end
40
+
41
+ desc "create", '生成网站'
42
+ option :name, default: "dev", desc: "设定文件夹名"
43
+ def create(type_string)
44
+ generator = Komic::Generator.new
45
+ builder = Builder::Factory.get_builder(type_string, options)
46
+ generator.create_website({ images: builder.images }, options)
47
+ end
41
48
  end
42
49
  end
@@ -5,6 +5,8 @@ require 'json'
5
5
  require 'json-schema'
6
6
  require 'mini_magick'
7
7
  require 'base64'
8
+ require 'open-uri'
9
+ require 'zip'
8
10
 
9
11
  require 'komic/version'
10
12
  require 'komic/utils'
@@ -123,24 +125,28 @@ module Komic
123
125
 
124
126
  [root_dir, image_dir].each { |path| FileUtils.mkdir_p path }
125
127
 
126
- files = data[:images]
128
+ images = data[:images]
127
129
 
128
- files.map.with_index do |image, index|
129
- image_manager = MiniMagick::Image.open(image[:src])
130
+ images.map.with_index do |image, index|
131
+ manager = MiniMagick::Image.open(image[:src])
130
132
 
131
- image_path = File.join(image_dir, [index, image_manager.type.downcase].join('.'))
132
- FileUtils.mv image[:src], image_path
133
+ image_path = File.join(image_dir,
134
+ [index, manager.type.downcase].join('.'))
135
+
136
+ manager.quality(60)
137
+ manager.strip()
138
+ manager.write image_path
133
139
  image[:src] = image_path
134
140
  image
135
141
  end
136
142
 
137
- thumbnails_builder = ThumbnailsBuilder.new(files)
143
+ thumbnails_builder = ThumbnailsBuilder.new(images)
138
144
  thumbnail_path = File.join(image_dir, './thumbnail.svg')
139
145
  File.open(thumbnail_path, 'w') do |file|
140
146
  file.write thumbnails_builder.to_build
141
147
  end
142
148
 
143
- files.map do |image, index|
149
+ images.map do |image, index|
144
150
  image[:src] = Utils.get_relative_path(image[:src], root_dir)
145
151
  if options[:'remote-url']
146
152
  image[:src] = "https://placeimg.com/#{image[:width]}/#{image[:height]}/any"
@@ -162,11 +168,25 @@ module Komic
162
168
  meta = Utils.deep_merge_hashes(meta, data[:meta])
163
169
  end
164
170
 
165
- content_builder = ContentBuilder.new(meta, files)
171
+ content_builder = ContentBuilder.new(meta, images)
166
172
  File.open(File.join(root_dir, './content.json'), 'w') do |file|
167
173
  file.write content_builder.to_build
168
174
  end
175
+ end
169
176
 
177
+ def create_website(data, options)
178
+ root_dir = File.join(Dir.pwd, options[:name])
179
+ create_package(data, options)
180
+ dist_project = "komic-web-dist"
181
+ dist_branch = "master"
182
+ uri = "https://github.com/komic-awesome/#{dist_project}/archive/#{dist_branch}.zip"
183
+ source = open(uri)
184
+ Zip::File.open(source.path) do |zip_file|
185
+ zip_file.each do |entry|
186
+ entry.extract(File.join(root_dir, File.basename(entry.name))) \
187
+ if File.fnmatch("#{dist_project}-#{dist_branch}/?*", entry.name)
188
+ end
189
+ end
170
190
  end
171
191
  end
172
192
  end
data/lib/komic/utils.rb CHANGED
@@ -45,5 +45,19 @@ module Komic
45
45
  Pathname.new(root)
46
46
  ))
47
47
  end
48
+
49
+ def create_progress(title, total)
50
+ # green background
51
+ color_code = "\e[0m\e[32m\e[7m\e[1m"
52
+ reset_code = "\e[0m"
53
+ progress_status = "#{color_code} %p%% #{reset_code}"
54
+
55
+ ProgressBar.create( :format => "%a %bᗧ%i #{progress_status} %t",
56
+ :title => title,
57
+ :progress_mark => ' ',
58
+ :remainder_mark => '・',
59
+ :total => total,
60
+ :starting_at => 0 )
61
+ end
48
62
  end
49
63
  end
data/lib/komic/version.rb CHANGED
@@ -1,4 +1,4 @@
1
1
  module Komic
2
- VERSION = '0.1.2'
2
+ VERSION = '0.1.3'
3
3
  CONTENT_JSON_VERSION = '0.1.0'
4
4
  end
@@ -0,0 +1,26 @@
1
+ require 'spec_helper'
2
+ require 'komic/builder'
3
+
4
+ describe Komic::Builder do
5
+ subject { Komic::Builder::Factory }
6
+ context "detect file" do
7
+ before { allow(File).to receive(:exists?).and_return(true) }
8
+ it "detect pdf type" do
9
+ expect( subject.detect_type('test.pdf') ).to be_eql('pdf')
10
+ expect{ subject.detect_type('.pdf') }.to raise_error RuntimeError
11
+ end
12
+ end
13
+
14
+ it "detect douban_album type" do
15
+ douban_album = "www.douban.com/photos/album/118525984/"
16
+
17
+ expect( subject.detect_type("http://#{douban_album}")).to \
18
+ be_eql('douban_album')
19
+ expect( subject.detect_type("https://#{douban_album}")).to \
20
+ be_eql('douban_album')
21
+
22
+ expect{ subject.detect_type("#{douban_album}") }.to raise_error RuntimeError
23
+ expect{ subject.detect_type('w.pdf') }.to raise_error RuntimeError
24
+ expect{ subject.detect_type('http://www.douban.com') }.to raise_error RuntimeError
25
+ end
26
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: komic-cli
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.2
4
+ version: 0.1.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - hxgdzyuyi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-08-06 00:00:00.000000000 Z
11
+ date: 2015-08-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: thor
@@ -108,6 +108,20 @@ dependencies:
108
108
  - - "~>"
109
109
  - !ruby/object:Gem::Version
110
110
  version: '1.7'
111
+ - !ruby/object:Gem::Dependency
112
+ name: rubyzip
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: '1.1'
118
+ type: :runtime
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: '1.1'
111
125
  - !ruby/object:Gem::Dependency
112
126
  name: bundler
113
127
  requirement: !ruby/object:Gem::Requirement
@@ -196,12 +210,15 @@ files:
196
210
  - bin/komic
197
211
  - komic-cli.gemspec
198
212
  - lib/komic.rb
213
+ - lib/komic/builder.rb
214
+ - lib/komic/builder/douban_album.rb
215
+ - lib/komic/builder/pdf.rb
199
216
  - lib/komic/cli.rb
200
- - lib/komic/crawler/douban.rb
201
217
  - lib/komic/generator/content.schema.json
202
218
  - lib/komic/generator/generator.rb
203
219
  - lib/komic/utils.rb
204
220
  - lib/komic/version.rb
221
+ - spec/builder_spec.rb
205
222
  - spec/spec_helper.rb
206
223
  - spec/utils_spec.rb
207
224
  homepage: https://rubygems.org/gems/komic-cli
@@ -229,5 +246,6 @@ signing_key:
229
246
  specification_version: 4
230
247
  summary: komic-cli
231
248
  test_files:
249
+ - spec/builder_spec.rb
232
250
  - spec/spec_helper.rb
233
251
  - spec/utils_spec.rb
@@ -1,101 +0,0 @@
1
- require 'mechanize'
2
- require 'fileutils'
3
- require 'mime/types'
4
- require 'tmpdir'
5
- require 'ruby-progressbar'
6
-
7
- module Komic
8
- module Crawler
9
- class Douban
10
- def initialize
11
- @mechanize = Mechanize.new
12
- @file_index = 0
13
- @tmpdir = Dir.mktmpdir
14
- @willbe_downloaded = []
15
- end
16
-
17
- def get_crawled_result(album_home_url)
18
- next_link_url = album_home_url
19
- next_link = nil
20
- album_title = nil
21
-
22
- @mechanize.get(album_home_url) do |page|
23
- album_title = page.at('title').text().strip!
24
- end
25
-
26
- loop do
27
- begin
28
- crawl_album_page(next_link_url)
29
- rescue Exception => e
30
- puts e.message
31
- puts e.backtrace.inspect
32
- end
33
-
34
- @mechanize.get(next_link_url) do |page|
35
- next_link = page.at('link[rel="next"]')
36
- end
37
- break if next_link.nil?
38
- next_link_url = next_link["href"]
39
- end
40
-
41
- # green background
42
- color_code = "\e[0m\e[32m\e[7m\e[1m"
43
- reset_code = "\e[0m"
44
- progress_status = "#{color_code} %p%% #{reset_code}"
45
-
46
- bar = ProgressBar.create( :format => "%a %bᗧ%i #{progress_status} %t",
47
- :title => 'Download image from douban',
48
- :progress_mark => ' ',
49
- :remainder_mark => '・',
50
- :total => @willbe_downloaded.size,
51
- :starting_at => 0 )
52
-
53
- image_pathes = @willbe_downloaded.map do |url|
54
- image_path = download_image url
55
- bar.increment
56
- image_path
57
- end
58
-
59
- return album_title, image_pathes
60
- end
61
-
62
- def crawl_album_page(album_page_url)
63
- @mechanize.get(album_page_url) do |page|
64
- page.search('.photolst_photo').each do |link|
65
- crawl_photo_page(link['href'])
66
- end
67
- end
68
- end
69
-
70
- def crawl_photo_page(photo_page_url)
71
- link_to_large = nil
72
- thumb_photo_url = nil
73
-
74
- @mechanize.get(photo_page_url) do |page|
75
- link_to_large = page.at('a[title="查看原图"]')
76
- thumb_photo_url = page.at('.image-show-inner img')["src"]
77
- end
78
-
79
- unless link_to_large.nil?
80
- @mechanize.get(link_to_large['href']) do |page|
81
- @willbe_downloaded.push(page.at('#pic-viewer img')["src"])
82
- end
83
- else
84
- @willbe_downloaded.push(thumb_photo_url)
85
- end
86
- end
87
-
88
- def download_image(photo_url)
89
- resource = @mechanize.get(photo_url)
90
- content_type = resource["content-type"]
91
- mime_type = MIME::Types[resource["content-type"]].first
92
- image_path = File.expand_path( \
93
- [@file_index, mime_type.extensions.first].join('.'), @tmpdir)
94
- resource.save(image_path)
95
- @file_index = @file_index + 1
96
- sleep 2
97
- image_path
98
- end
99
- end
100
- end
101
- end