ficrip 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 274aeb5446a0a3d556d227c02e66fd394c0f7c73
-  data.tar.gz: 95c5d132493981015636a876efd8fc139a800e3a
+  metadata.gz: 1a777da71d255f1d0dd702848a84d4442f5e8f54
+  data.tar.gz: e7b03da00e5e7aa55fda7c3b2771bb60c4061eb7
 SHA512:
-  metadata.gz: e9be42f1da6348b7d24c348aae7e82864c3b8f1300041201c516ca1d007f2db82441a45c15560ed61b39329b728764e44181b290127dc4516b429761c085de09
-  data.tar.gz: 84b991e21709287b39ef532e611910615922c1709992914e6dd149c4dccfbdc224ba145a68cdfef16b384ec78d3f499c69dad41fb1590840caaacd7c06aedcfa
+  metadata.gz: d22c5f1d7d942af2c2f0eaacce64877b32b816e209215ac4818e26e730d809422419a60322fce4b8e7e5ef60f8b93e2cc5639d5fa496028b71421da72a772206
+  data.tar.gz: c0e711f7c43c3e826f16b274384efb5f09d41d27358447f9e5598899b769e4911a20d12950a2f83fcfe9f3c20faa1b64a2311b500366836871c091521ba28d51

data/bin/ficrip

@@ -53,7 +53,8 @@ storyids.each_with_index do |storyid, idx|
   progressbar.total = fic.chapters.count
   progressbar.format = prefix + "#{author_title} | %a [%B] %p%% (%c/%C)"
 
-  epub = fic.bind(callback: lambda { progressbar.increment })
+
+  epub = fic.bind(version: opts[:epub2] ? 2 : 3, callback: lambda { progressbar.increment })
 
   progressbar.format = prefix + "#{author_title}... Binding"
   filename = File.join(Dir.pwd, "#{author_title}.epub")

data/ficrip.gemspec

@@ -24,8 +24,10 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'contracts', '~> 0.14'
   spec.add_dependency 'i18n_data', '~> 0.7'
   spec.add_dependency 'fastimage', '~> 1.8'
+  spec.add_dependency 'chronic_duration', '~> 0.10'
+  spec.add_dependency 'retryable', '~> 2.0'
 
-  spec.add_development_dependency 'bundler', '~> 1.12'
-  spec.add_development_dependency 'rake', '~> 11.2'
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec', '~> 3.5'
 end

data/lib/ficrip/extensions.rb

@@ -1,6 +1,7 @@
 class Array
   def find_with(str)
-    find { |i| i.start_with? str }.gsub(str, '').strip
+    r = find { |i| i.start_with? str }
+    r.gsub(str, '').strip if r
   end
 end
 
@@ -14,3 +15,30 @@ class String
     gsub(/^[ \t]{#{indent}}/, '')
   end
 end
+
+class Object
+  def as
+    yield self
+  end
+
+  # From http://stackoverflow.com/a/8206537
+  def deep_clone
+    return @deep_cloning_obj if @deep_cloning
+    @deep_cloning_obj = clone
+    @deep_cloning_obj.instance_variables.each do |var|
+      val = @deep_cloning_obj.instance_variable_get(var)
+      begin
+        @deep_cloning = true
+        val = val.deep_clone
+      rescue TypeError
+        next
+      ensure
+        @deep_cloning = false
+      end
+      @deep_cloning_obj.instance_variable_set(var, val)
+    end
+    deep_cloning_obj = @deep_cloning_obj
+    @deep_cloning_obj = nil
+    deep_cloning_obj
+  end
+end

data/lib/ficrip/process.rb

@@ -1,8 +1,12 @@
 require 'contracts'
-require 'gepub'
+require 'open-uri'
+
+# Helpers
 require 'i18n_data'
 require 'fastimage'
-require 'open-uri'
+require 'chronic_duration'
+require 'retryable'
+
 require_relative 'extensions'
 
 module Ficrip
@@ -12,7 +16,12 @@ module Ficrip
   Contract Integer => Story
   def self.fetch(storyid)
     base_url     = "https://www.fanfiction.net/s/#{storyid}/"
-    primary_page = Nokogiri::HTML open(base_url)
+
+    primary_page = Retryable.retryable(tries: :infinite, on: OpenURI::HTTPError) do
+      Nokogiri::HTML open(base_url)
+    end
+
+    raise(ArgumentError.new("Invalid StoryID #{storyid}")) if primary_page.css('#profile_top').count == 0
 
     title  = primary_page.css('#profile_top > b').first.text
     author = primary_page.css('#profile_top > a').first.text
@@ -23,18 +32,33 @@ module Ficrip
 
       info = primary_page.css('#profile_top > span.xgray.xcontrast_txt').text.split(' - ')
 
-      s.rating         = info.find_with 'Rated: Fiction'
-      s.language       = info[1]
-      s.genres         = info[2].split('/')
-      s.characters     = info[3].strip
-      s.chapter_count  = info.find_with('Chapters:').parse_int
-      s.word_count     = info.find_with('Words:').parse_int
-      s.review_count   = info.find_with('Reviews:').parse_int
-      s.favs_count     = info.find_with('Favs:').parse_int
-      s.follows_count  = info.find_with('Follows:').parse_int
-      s.updated_date   = Date.parse info.find_with('Updated:')
-      s.published_date = Date.parse info.find_with('Published:')
-      s.info_id        = info.find_with('id:').to_i
+      s.rating        = info.find_with 'Rated: Fiction'
+      s.language      = info[1]
+      s.genres        = info[2].split('/')
+      s.characters    = info[3].strip
+      s.chapter_count = info.find_with('Chapters:').as { |c| c.parse_int unless c.nil? }
+      s.word_count    = info.find_with('Words:').parse_int
+      s.review_count  = info.find_with('Reviews:').parse_int
+      s.favs_count    = info.find_with('Favs:').parse_int
+      s.follows_count = info.find_with('Follows:').parse_int
+
+      s.updated_date = info.find_with('Updated:').as do |d|
+        begin
+          Date.strptime(d, '%m/%d/%Y')
+        rescue
+          Date.strptime(d, '%m/%d') rescue (Time.now - ChronicDuration.parse(d)).to_date
+        end if d
+      end
+
+      s.published_date = info.find_with('Published:').as do |d|
+        begin
+          Date.strptime(d, '%m/%d/%Y')
+        rescue
+          Date.strptime(d, '%m/%d') rescue (Time.now - ChronicDuration.parse(d)).to_date
+        end
+      end
+
+      s.info_id = info.find_with('id:').to_i
 
       raise(Exception.new("Error! StoryID and parsed ID don't match.")) if s.info_id != storyid
 
@@ -46,14 +70,14 @@ module Ficrip
       if chapter_select
         s.chapters = chapter_select.children.map(&:text)
       else
-        s.chapters = "1. #{title}"
+        s.chapters = ["1. #{title}"]
       end
     end
   end
 
   Contract Integer => GEPUB::Book
   def self.get(storyid, version: 3)
-    fetch(storyid).bind(version)
+    fetch(storyid).bind(version: version)
   end
 
 end

data/lib/ficrip/story.rb

@@ -1,5 +1,8 @@
 require 'contracts'
 require 'open-uri'
+require 'retryable'
+require 'gepub'
+
 require_relative 'extensions'
 
 module Ficrip
@@ -7,7 +10,7 @@ module Ficrip
     include Contracts::Core
     include Contracts::Builtin
 
-    attr_accessor :title, :author, :metadata
+    attr_accessor :title, :author, :url, :metadata
 
     DOCTYPE = {
         3 => '<!DOCTYPE html>',
@@ -21,11 +24,6 @@ module Ficrip
       @metadata = metadata
     end
 
-    def construct
-      yield self
-      self
-    end
-
     def self.construct(title, author, url)
       s = Story::new(title, author, url)
       yield s
@@ -43,7 +41,14 @@ module Ficrip
       end
     end
 
+    def respond_to_missing?(method_sym, include_private = false)
+      @metadata.key?(method_sym) ||
+          (method_sym != :title= && method_sym != :author= && method_sym.to_s.end_with?('=')) ||
+          super
+    end
+
     Contract Symbol, String => Story
+
     def add_metadata(key, value)
       @metadata[key] = value
       self
@@ -51,8 +56,6 @@ module Ficrip
 
     # Contract { version: Maybe[Or[2, 3]] }
     def bind(version: 3, callback: nil)
-
-
       book = GEPUB::Book.new('OEPBS/package.opf', 'version' => version.to_f.to_s)
       book.primary_identifier(@url, 'BookId', 'URL')
 
@@ -62,7 +65,7 @@ module Ficrip
 
       # Cover if it exists
       if @metadata.key? :cover_url
-        cover = open(@metadata[:cover_url], 'Referer' => @url)
+        cover      = open!(@metadata[:cover_url], 'Referer' => @url)
         cover_type = FastImage.type(cover)
         book.add_item(format('img/cover_image.%s', cover_type), cover)
             .cover_image
@@ -91,18 +94,49 @@ module Ficrip
         XHTML
       end
 
+      titlepage = <<-XHTML.strip_heredoc
+        <?xml version="1.0" encoding="utf-8"?>
+        #{DOCTYPE[version]}
+        <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
+          <head>
+            <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
+            <title>#{@title}</title>
+            <style type="text/css" title="override_css">
+              .outer { display: table; height: 75%; width: 100%; }
+              .middle { display: table-cell; vertical-align: middle; }
+              .inner { text-align: center; }
+            </style>
+          </head>
+          <body>
+            <div class="outer"><div class="middle"><div class="inner">
+              <h1>#{@title}</h1>
+              <h3>#{@author}</h3>
+            </div></div></div>
+          </body>
+        </html>
+      XHTML
+
+
+      table_of_contents = nil
       book.ordered do
         book.add_item('img/coverpage.xhtml')
             .add_content(StringIO.new(coverpage))
             .toc_text(@title) if @metadata.key? :cover_url
 
-        unless @metadata[:chapters]
-          @metadata[:chapters] = @url
-        end
+        book.add_item('text/titlepage.xhtml')
+            .add_content(StringIO.new(Nokogiri::XML(titlepage) { |c| c.noblanks }.to_xhtml(indent: 2)))
+
+        book.add_item('text/infopage.xhtml')
+            .add_content(StringIO.new(Nokogiri::XML(render_metadata) { |c| c.noblanks }.to_xhtml(indent: 2)))
+            .toc_text('About')
+
+        # We want our TOC to be after the cover and titlepage, but we don't any content
+        # for it yet, so we save it for later.
+        table_of_contents = book.add_item('text/toc.xhtml').toc_text('Table of Contents')
 
         chapters.each do |chapter|
           chapter_num, chapter_title = chapter.match(/^(\d+)\s*[-\\.)]?\s+(.*)/).captures
-          chapter_page               = Nokogiri::HTML open(URI.join(@url, chapter_num))
+          chapter_page               = Nokogiri::HTML open!(URI.join(@url, chapter_num))
 
           storytext = chapter_page.css('#storytext').first
           storytext.xpath('//@noshade').remove
@@ -120,20 +154,104 @@ module Ficrip
                 #{'<section epub:type="chapter">' if version == 3}
                   <h1 style="text-align:center">#{chapter_title}</h1>
                   #{storytext.children.to_xhtml}
-                #{'</section>' if version == 3}
+          #{'</section>' if version == 3}
               </body>
             </html>
           XHTML
 
-          book.add_item(format('text/chapter%03d.xhtml', chapter_num),
-                        nil, "c#{chapter_num}")
-              .add_content(StringIO.new(Nokogiri::XML(chapter_xhtml){|c| c.noblanks}.to_xhtml(indent:2)))
+          book.add_item(format('text/chapter%03d.xhtml', chapter_num), nil, "c#{chapter_num}")
+              .add_content(StringIO.new(Nokogiri::XML(chapter_xhtml) { |c| c.noblanks }.to_xhtml(indent: 2)))
               .toc_text(chapter_title)
 
-          callback.call if callback
+          if callback
+            args = [chapter_num.to_i, chapters.count]
+            n    = callback.arity
+            callback.call *(n < 0 ? args : args.take(n))
+          end
         end
       end
+
+      # This generates a proper Table of Contents page at the start of the book by
+      # removing references to the cover and TOC itself
+      book_copy = book.deep_clone
+      cut_idx   = @metadata.key?(:cover_url) ? 3 : 2
+      book_copy.instance_variable_set(:@toc, book_copy.instance_variable_get(:@toc)[cut_idx..-1])
+      table_of_contents.add_content(StringIO.new(book_copy.nav_doc)) # Finally, we get to add the actual content
+
+      # Now that we've generated the TOC page, go through every chapter reference in the
+      # toc and prepend the chapter number. This is for the epub's built-in table of contents
+      book.instance_variable_get(:@toc)[cut_idx..-1].each_with_index do |chap, idx|
+        chap[:text] = "#{idx + 1}. #{chap[:text]}"
+      end
+
+      add_item('nav.html', StringIO.new(book_copy.nav_doc), 'nav').add_property('nav') if version == 3
+
       book
     end
+
+    def render_metadata
+      data = {
+          'Rating'              => rating,
+          'Language'            => language,
+          'Genres'              => genres.join(', '),
+          'Characters/Pairings' => characters,
+          'Chapter count'       => format_num(chapter_count),
+          'Word count'          => format_num(word_count),
+          'Reviews'             => "<a href='https://fanfiction.com/r/#{info_id}'>" + format_num(review_count) + '</a>',
+          'Favorites'           => format_num(favs_count),
+          'Follows'             => format_num(follows_count),
+          'Updated'             => updated_date,
+          'Published'           => published_date,
+          'ID'                  => info_id
+      }
+
+      Nokogiri::XML::Builder.new(encoding: 'utf-8') { |doc|
+        doc.html('xmlns' => 'http://www.w3.org/1999/xhtml', 'xml:lang' => 'en') {
+          doc.head {
+            doc.meta 'http-equiv' => 'Content-Type', 'content' => 'text/html; charset=UTF-8'
+            doc.title 'About'
+          }
+          doc.body {
+            doc.p { doc.strong 'Author: '; doc.a(href: author_url) { doc.text @author } }
+            doc.p { doc.strong 'Summary:'; doc.br; doc.text summary }
+            data.each do |k, v|
+              doc.span {
+                doc.strong(k + ':')
+                (v.to_s.start_with? '<a') ? doc << v.to_s : doc.text(' ' + v.to_s)
+                doc.br
+              }
+            end
+          }
+        }
+      }.to_xml
+    end
+
+    private
+    def open!(*args, &block)
+      Retryable.retryable(tries: :infinite, on: OpenURI::HTTPError) do
+        open(*args, &block)
+      end
+    end
+
+    def handle_url(string)
+      if string.start_with? 'https://www.fanfiction.net/u/'
+        "<a href='#{string}'>"
+      end
+    end
+
+    # # File activesupport/lib/active_support/inflector/methods.rb, line 123
+    # def humanize(word, capitalize: false)
+    #   result = word.to_s.dup
+    #   result.sub!(/\A_+/, ''.freeze)
+    #   result.sub!(/_id\z/, ''.freeze)
+    #   result.tr!('_'.freeze, ' '.freeze)
+    #   result.gsub!(/([a-z\d]*)/i) { |match| match.downcase }
+    #   result.sub!(/\A\w/) { |match| match.upcase } if capitalize
+    #   result
+    # end
+
+    def format_num(num)
+      num.to_s.reverse.gsub(/...(?=.)/, '\&,').reverse
+    end
   end
-end
+end

data/lib/ficrip/version.rb

@@ -1,3 +1,3 @@
 module Ficrip
-  VERSION = '0.0.1'
+  VERSION = '0.0.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ficrip
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Katherine Whitlock
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-21 00:00:00.000000000 Z
+date: 2016-07-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -109,33 +109,61 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.8'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: chronic_duration
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.12'
-  type: :development
+        version: '0.10'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.12'
+        version: '0.10'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: retryable
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '11.2'
-  type: :development
+        version: '2.0'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '11.2'
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement