comic_walker 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 31c86d5ee6917c1d2443b444ad0af64ea30312b7
-  data.tar.gz: ad9a19cdaba4cc3ef9e78caaf4283e1024e25e1e
+  metadata.gz: 7fe84087441847fad172320a68b0a4849d5ccb0f
+  data.tar.gz: ad4627cd74295d5e64c457d74729cd379dedb10f
 SHA512:
-  metadata.gz: 484678c62454decd50c4153b0f3b9f2ab5e018f4cf08d26ffa61196a3d3132968a1c6cf09df458aef884a179874936c90e4d3a5fdadcb18989eaee0b83d56034
-  data.tar.gz: 0265eec741fde6d0d8a647d4023ab5578e66d3e6917341295fa5fe53c8d414606965a12225da895d2dedb9bdad3c688555b2be4cf54f4d16c5358e368de2ccce
+  metadata.gz: 7ca5b14ea61e4a6118ee906b64e7dff21b95ec07b9c9e70827b765d6952fb8d824973bb1ba50ffe8ed918a40030d2f625d29621d41a69d2380cadc43d4bedbcf
+  data.tar.gz: 62076faf1ff081f17c5f42e5e9cba25949ad523eb87c3e2b54fda226524cfdd7b9e4be0bd6db98670776eca38666ea4a63c8b9c576c05020650989c1ca68a14c

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.3.0 (2014-05-12)
+- Download configuration_pack and images directly
+    - Drop jar file download
+    - Drop base64-encoded image decoder
+    - Many internal method changes
+
 # 0.2.2 (2014-05-11)
 - Fix image fetch errors for recent comics
 

data/comic_walker.gemspec

@@ -23,7 +23,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "yard"
   spec.add_dependency "addressable"
   spec.add_dependency "http-cookie"
+  spec.add_dependency "net-http-persistent"
+  spec.add_dependency "retryable"
   spec.add_dependency "rmagick"
-  spec.add_dependency "rubyzip"
   spec.add_dependency "thor"
 end

data/lib/comic_walker/cli.rb

@@ -4,7 +4,7 @@ require 'pp'
 require 'thor'
 require 'yaml'
 require 'comic_walker/client'
-require 'comic_walker/end_layer_decoder'
+require 'comic_walker/content_downloader'
 require 'comic_walker/item_decoder'
 require 'comic_walker/v1/client'
 
@@ -20,7 +20,7 @@ module ComicWalker
     def save(*cids)
       client = Client.new
       cids.each do |cid|
-        save_content(client, cid)
+        ContentDownloader.new(client, cid).save
       end
     end
 
@@ -45,9 +45,7 @@ module ComicWalker
       jar = HTTP::CookieJar.new
       load_cookies(jar)
       client = V1::Client.new(jar, load_uuid)
-      json = client.start do
-        client.contents(per_page: options[:per_page], page: options[:page])
-      end
+      json = client.contents(per_page: options[:per_page], page: options[:page])
       save_cookies(jar)
 
       json['contents'].sort_by { |content| Time.parse(content['updated_at']) }.reverse_each do |content|
@@ -72,48 +70,6 @@ module ComicWalker
 
     private
 
-    def save_content(client, cid)
-      license = client.get_license(cid)
-      info = license.get_info.first
-      last_update = Time.strptime(info['last_update'], '%Y%m%d%H%M')
-      title = info['issues'].first['content_name']
-
-      license.with_jar do |zip|
-        pages = []
-        items = {}
-        decoder = nil
-        while entry = zip.get_next_entry
-          case entry.name
-          when 'configuration_pack.json'
-            config = JSON.parse(zip.read)
-            decoder = ItemDecoder.new(config)
-          when /end_layer\.json/
-            begin
-              pp EndLayerDecoder.new(license.key).decode(zip.read)
-            rescue JSON::ParserError
-              $stderr.puts "WARNING: #{cid}: Could not decode end_layer.json.enc"
-            end
-          when %r{\Aitem/}
-            items[entry.name] = zip.read
-          end
-        end
-
-        img_dir = Pathname.new(title).join(cid).tap(&:mkpath)
-        decoder.pages.each.with_index do |file, i|
-          dat_path = Pathname.new(file).join('0.dat')
-          img_fname = dat_path.parent.basename.sub_ext('.jpg')
-          img_path = img_dir.join(sprintf('%03d_%s', i, img_fname))
-
-          if data = items[dat_path.to_s]
-            decoder.decode_b64(file, dat_path, img_path, data)
-          else
-            decoder.decode(file, dat_path, img_path, license.get_jpeg(file))
-          end
-          puts "#{dat_path} -> #{img_path}"
-        end
-      end
-    end
-
     def load_cookies(jar)
       if COOKIE_PATH.readable?
         jar.load(COOKIE_PATH.to_s, session: true)
@@ -147,20 +103,18 @@ module ComicWalker
       page = 1
       per_page = 200
       child_cids = {}
-      client.start do
-        until child_cids.size == parent_cids do
-          json = client.contents(page: page, per_page: per_page)
-          contents = json['contents']
-          if contents.empty?
-            break
-          end
-          contents.each do |c|
-            if parent_cids.include?(c['content_id'])
-              child_cids[c['content_id']] = c['sub_contents']
-            end
+      until child_cids.size == parent_cids do
+        json = client.contents(page: page, per_page: per_page)
+        contents = json['contents']
+        if contents.empty?
+          break
+        end
+        contents.each do |c|
+          if parent_cids.include?(c['content_id'])
+            child_cids[c['content_id']] = c['sub_contents']
           end
-          page += 1
         end
+        page += 1
       end
       parent_cids.each do |cid|
         unless child_cids.has_key?(cid)

data/lib/comic_walker/client/license.rb

@@ -1,23 +1,19 @@
-require 'addressable/uri'
+require 'uri'
 require 'base64'
-require 'net/http'
-require 'zip'
+require 'net/http/persistent'
 
 module ComicWalker
   class Client
     class License
       def initialize(json)
         @json = json
+        @http = Net::HTTP::Persistent.new('comic_walker')
       end
 
       def agreement
         @json['agreement']
       end
 
-      def jar_url
-        url_prefix + agreement['jar_file_name']
-      end
-
       def info_url
         url_prefix + agreement['info_file_name']
       end
@@ -26,31 +22,26 @@ module ComicWalker
         agreement['url_prefix']
       end
 
-      def key
-        Base64.decode64(agreement['key'])
+      CONFIGURATION_PACK_FILENAME = 'configuration_pack.json'
+
+      def configuration_pack_url
+        url_prefix + CONFIGURATION_PACK_FILENAME
+      end
+
+      def get(url)
+        @http.request(URI.parse(url))
       end
 
-      def with_jar(&block)
-        uri = Addressable::URI.parse(jar_url)
-        body = Net::HTTP.start(uri.host) do |http|
-          http.get(uri.request_uri).body
-        end
-        Zip::InputStream.open(StringIO.new(body), &block)
+      def get_configuration_pack
+        JSON.parse(get(configuration_pack_url).body)
       end
 
       def get_info
-        uri = Addressable::URI.parse(info_url)
-        body = Net::HTTP.start(uri.host) do |http|
-          http.get(uri.request_uri).body
-        end
-        JSON.parse(body)
+        JSON.parse(get(info_url).body)
       end
 
       def get_jpeg(file)
-        uri = Addressable::URI.parse(url_prefix + file + '/0.jpeg')
-        Net::HTTP.start(uri.host) do |http|
-          http.get(uri.request_uri).body
-        end
+        get(url_prefix + file + '/0.jpeg').body
       end
     end
   end

data/lib/comic_walker/content_downloader.rb

@@ -0,0 +1,27 @@
+module ComicWalker
+  class ContentDownloader
+    def initialize(client, cid)
+      @client = client
+      @cid = cid
+    end
+
+    def save
+      license = @client.get_license(@cid)
+      decoder = ItemDecoder.new(license.get_configuration_pack)
+      img_dir = image_dir(license)
+      decoder.pages.each.with_index do |file, i|
+        dat_path = Pathname.new(file).join('0.dat')
+        img_fname = dat_path.parent.basename.sub_ext('.jpg')
+        img_path = img_dir.join(sprintf('%03d_%s', i, img_fname))
+        decoder.decode(file, dat_path, img_path, license.get_jpeg(file))
+        puts "#{dat_path} -> #{img_path}"
+      end
+    end
+
+    def image_dir(license)
+      info = license.get_info.first
+      title = info['issues'].first['content_name']
+      Pathname.new(title).join(@cid).tap(&:mkpath)
+    end
+  end
+end

data/lib/comic_walker/item_decoder.rb

@@ -5,18 +5,7 @@ module ComicWalker
   class ItemDecoder
     attr_reader :pages
 
-    DEFAULT_CT = '20000101000000'
-    DEFAULT_ST = '20000101000000'
-    DEFAULT_ET = '99991231235959'
-
     def initialize(configuration_pack)
-      fname = '0.dat'
-      ct = configuration_pack['ct'] || DEFAULT_CT
-      st = configuration_pack['st'] || DEFAULT_ST
-      et = configuration_pack['et'] || DEFAULT_ET
-      @key1 = (ct + st + fname).unpack('C*')
-      @key2 = (ct + fname + et).unpack('C*')
-      @key3 = (fname + st + et).unpack('C*')
       @pages = []
       configuration_pack['configuration']['contents'].each do |content|
         pages[content['index']-1] = content['file']
@@ -27,19 +16,6 @@ module ComicWalker
       end
     end
 
-    def decode_b64(file, dat_path, img_path, b64data)
-      bs = 128
-      hs = 1024
-      blob = Unknown.finish(@key1, hs,
-        Unknown.decrypt(@key2, bs,
-          Unknown.prepare(@key3,
-            Base64.decode64(b64data).unpack('C*')
-          )
-        )
-      ).pack('C*')
-      decode(file, dat_path, img_path, blob)
-    end
-
     def decode(file, dat_path, img_path, blob)
       src = Magick::Image.from_blob(blob).first
       width = src.columns

data/lib/comic_walker/item_decoder/unknown.rb

@@ -5,47 +5,6 @@ module ComicWalker
     module Unknown
       module_function
 
-      # @param [Array<Fixnum>] key
-      # @param [Array<Fixnum>] data Encrypted data
-      # @return [Array<Fixnum>]
-      def prepare(key, data)
-        s = Cipher.gen_rc4_table(key)
-        data.map.with_index do |b, i|
-          b ^ s[i % 256]
-        end
-      end
-
-      # @param [Array<Fixnum>] key
-      # @param [Integer] bsize block size?
-      # @param [Array<Fixnum>] data Encrypted data
-      # @return [Array<Fixnum>] Chunked decrypted data
-      def decrypt(key, bsize, data)
-        s = []
-        0.step(data.size-1, bsize) do |i|
-          s << data[i]
-        end
-
-        c = Cipher.decrypt_rc4(key, s)
-        # s.size == c.size
-
-        0.step(data.size-1, bsize) do |i|
-          data[i] = c.shift
-        end
-        data
-      end
-
-      # @param [Array<Fixnum>] key
-      # @param [Integer] hsize header size?
-      # @param [Array<Fixnum>] data Encrypted data
-      # @return [Array<Fixnum>] data
-      def finish(key, hsize, data)
-        hsize = [hsize, data.size].min
-        Cipher.decrypt_rc4(key, data.slice(0, hsize)).each.with_index do |x, i|
-          data[i] = x
-        end
-        data
-      end
-
       # Calculate moves.
       # @param [Integer] width Width of the image
       # @param [Integer] height Height of the image

data/lib/comic_walker/v1/client.rb

@@ -1,7 +1,9 @@
 require 'addressable/uri'
 require 'http-cookie'
 require 'json'
-require 'net/http'
+require 'net/http/persistent'
+require 'retryable'
+require 'uri'
 
 module ComicWalker
   module V1
@@ -9,19 +11,11 @@ module ComicWalker
       BASE_URI = Addressable::URI.parse("https://cnts.comic-walker.com")
 
       def initialize(jar, uuid)
-        @https = Net::HTTP.new(BASE_URI.host, 443)
-        @https.use_ssl = true
-        @https.verify_mode = OpenSSL::SSL::VERIFY_PEER
+        @https = Net::HTTP::Persistent.new('comic_walker')
         @jar = jar
         @uuid = uuid
       end
 
-      def start(&block)
-        @https.start do
-          block.call
-        end
-      end
-
       AID = 'KDCWI_JP'
       AVER = '1.2.0'
 
@@ -31,8 +25,8 @@ module ComicWalker
       end
 
       def create_session
-        retried = 0
-        begin
+        on_exception = lambda { |exception| create_user }
+        retryable(tries: 2, on: UnknownDeviceError, sleep: 0, exception_cb: on_exception) do
           res = post('/user_sessions/create', {
             DID: @uuid,
             PIN: @uuid,
@@ -47,14 +41,6 @@ module ComicWalker
           else
             JSON.parse(res.body)
           end
-        rescue UnknownDeviceError => e
-          if retried == 0
-            retried += 1
-            create_user
-            retry
-          else
-            raise e
-          end
         end
       end
 
@@ -68,7 +54,6 @@ module ComicWalker
       end
 
       def contents(params = {})
-        retried = 0
         params = {
           AID: AID,
           AVER: AVER,
@@ -80,7 +65,8 @@ module ComicWalker
           languages: 'ja',
         }.merge(params)
 
-        begin
+        on_exception = lambda { |exception| create_session }
+        retryable(tries: 2, on: NoValidSessionError, sleep: 0, exception_cb: on_exception) do
           res = get('/v1/contents', params)
           case res.body
           when 'NoValidSessionError'
@@ -88,14 +74,6 @@ module ComicWalker
           else
             JSON.parse(res.body)
           end
-        rescue NoValidSessionError => e
-          if retried == 0
-            retried += 1
-            create_session
-            retry
-          else
-            raise e
-          end
         end
       end
 
@@ -117,7 +95,7 @@ module ComicWalker
 
       def request_with_cookie(uri, req)
         req['cookie'] = HTTP::Cookie.cookie_value(@jar.cookies(uri.to_s))
-        @https.request(req).tap do |res|
+        @https.request(URI.parse(uri.to_s), req).tap do |res|
           @jar.parse(res['set-cookie'], uri.to_s)
         end
       end

data/lib/comic_walker/version.rb

@@ -1,5 +1,5 @@
 module ComicWalker
-  VERSION = "0.2.2"
+  VERSION = "0.3.0"
 
   VIEWER_VERSION = "1.0.5_2014-04-11"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: comic_walker
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - Kohei Suzuki
@@ -81,7 +81,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rmagick
+  name: net-http-persistent
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -95,7 +95,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rubyzip
+  name: retryable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rmagick
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -143,7 +157,7 @@ files:
 - lib/comic_walker/cli.rb
 - lib/comic_walker/client.rb
 - lib/comic_walker/client/license.rb
-- lib/comic_walker/end_layer_decoder.rb
+- lib/comic_walker/content_downloader.rb
 - lib/comic_walker/item_decoder.rb
 - lib/comic_walker/item_decoder/unknown.rb
 - lib/comic_walker/v1/client.rb

data/lib/comic_walker/end_layer_decoder.rb

@@ -1,14 +0,0 @@
-require 'json'
-require 'comic_walker/cipher'
-
-module ComicWalker
-  class EndLayerDecoder
-    def initialize(key)
-      @key = key
-    end
-
-    def decode(b64)
-      JSON.parse(Cipher.decrypt_b64(@key, b64))
-    end
-  end
-end