ffnpdf 0.0.2

data/.gitignore

@@ -0,0 +1,7 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+.*.sw[a-z]
+*.un~
+Session.vim

data/.rspec

@@ -0,0 +1,2 @@
+--format nested
+--color

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in ffnpdf.gemspec
+gemspec

data/README.md

@@ -0,0 +1,65 @@
+# FFNPDF
+
+Just scratched an coding itch I've had for quite some time.
+
+This gem scrapes stories from FanFiction.net and generates LaTeX PDFs. Polished turds, anyone?
+
+## Installation
+
+To install this stupid gem:
+
+    gem install ffnpdf
+
+It also requires [pandoc](http://johnmacfarlane.net/pandoc/) as well as a bunch of extra stuff related to LaTeX/XeTeX. In Ubuntu, you can install this using the following command:
+
+    sudo apt-get install pandoc texlive-latex-extra texlive-xetex
+
+It also uses [DejaVu Fonts](http://dejavu-fonts.org/wiki/Main_Page) as the default font. This font family is installed by default in Ubuntu.
+
+## Usage
+
+The basic command only requires the story's story ID.
+
+    ffnpdf [STORY_ID]
+
+This will scrape FFN for the story, convert the chapters into markdown, combine all chapters into a single file, and converts it into PDF, all inside the `STORY_ID` folder.
+
+## Additional Commands
+
+Since you may want to do your own edits to the fics (e.g. remove authors notes, correct chapter headers) there are additional commands that allow you to perform individual steps of the process.
+
+### Pull
+
+    ffnpdf pull [STORY_ID]
+
+This command 
+
+* creates the `STORY_ID` folder
+* pulls chapters and converts them to Markdown
+* creates template and variables (config) file
+
+The markdown files are numbered from `0001.md` onwards. A title section, `0000.md`, is also created.
+
+The `xetex.template` file is obviously the XeTeX template, while the `variables.txt` defines the [template variables section](http://johnmacfarlane.net/pandoc/README.html#general-writer-options) used when generating the PDF. Both can be modified to affect the result of running both `ffnpdf build` and `ffnpdf convert`.
+
+
+### Build
+
+    ffnpdf build [STORY_ID]
+
+This command 
+
+* combines all `.md` files to a single file, `combined.md`, in alphabetical order
+* converts `combined.md` to `combined.pdf` while using both the template and variables file
+
+### Build
+
+    ffnpdf convert [STORY_ID]
+
+This command does only the second step of the Build i.e. converts `combined.md` to `combined.pdf`.
+
+## Troubleshooting
+
+**Table of Contents don't match the actual pages of the chapters.**
+
+[This is a LaTeX issue](http://tex.stackexchange.com/a/30946). Just rerun the `ffnpdf convert` command to rebuild the PDF.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/autotest/discover.rb

@@ -0,0 +1 @@
+Autotest.add_discovery { "rspec2" }

data/bin/ffnpdf

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
+require 'ffnpdf'
+
+Ffnpdf::Exec.new(ARGV)

data/ffnpdf.gemspec

@@ -0,0 +1,29 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "ffnpdf/version"
+
+Gem::Specification.new do |s|
+  s.name        = "ffnpdf"
+  s.version     = Ffnpdf::VERSION
+  s.authors     = ["Bryan Bibat"]
+  s.email       = ["bry@bryanbibat.net"]
+  s.homepage    = "https://github.com/bryanbibat/ffnpdf"
+  s.summary     = %q{PDF generator for FF.net stories}
+  s.description = %q{Scrapes a story off FF.net, converts it to Markdown, and turns it to PDF with LaTeX}
+
+  s.rubyforge_project = "ffnpdf"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "test_notifier"
+  s.add_development_dependency "autotest"
+  s.add_development_dependency "nokogiri"
+  s.add_development_dependency "httparty"
+  s.add_runtime_dependency "nokogiri"
+  s.add_runtime_dependency "httparty"
+end

data/lib/ffnpdf.rb

@@ -0,0 +1,8 @@
+require "ffnpdf/version"
+require "ffnpdf/exec"
+require "ffnpdf/story"
+require "ffnpdf/converter"
+
+module Ffnpdf
+  # Your code goes here...
+end

data/lib/ffnpdf/converter.rb

@@ -0,0 +1,16 @@
+module Ffnpdf
+  class Converter
+    def self.exec(command)
+      `#{command}`
+    end
+
+    def self.convert_to_pdf(story_id)
+      Dir.chdir story_id
+      
+      command = "markdown2pdf --xetex --template=xetex.template --toc #{IO.readlines("variables.txt")[0].strip} combined.md"
+      Converter.exec(command)
+
+      Dir.chdir "../" 
+    end
+  end
+end

data/lib/ffnpdf/exec.rb

@@ -0,0 +1,57 @@
+module Ffnpdf
+  class Exec
+    def initialize(argv)
+      invalid = false
+      if argv.size == 0
+        invalid = true
+      elsif /^[\d]+$/.match argv[0]
+        story = Ffnpdf::Story.new(argv[0])
+        if story.check_story
+          story.pull_story
+          story.build_story
+        else
+          $stderr.puts story.error
+        end
+      elsif argv[0] == "pull" and /^[\d]+$/.match argv[1]
+        story = Ffnpdf::Story.new(argv[1])
+        if story.check_story
+          story.pull_story
+        else
+          $stderr.puts story.error
+        end
+      elsif argv[0] == "build" and /^[\d]+$/.match argv[1]
+        story = Ffnpdf::Story.new(argv[1])
+        if story.check_story_dir
+          story.build_story
+        else
+          $stderr.puts story.error
+        end
+      elsif argv[0] == "convert" and /^[\d]+$/.match argv[1]
+        story = Ffnpdf::Story.new(argv[1])
+        if story.check_story_dir
+          story.convert_to_pdf
+        else
+          $stderr.puts story.error
+        end
+      else
+        invalid = true
+      end
+
+      if invalid
+        $stderr.puts <<-MSG
+Usage
+  ffnpdf [STORY_ID]
+  ffnpdf [COMMAND] [STORY_ID]
+
+Description
+  Scrapes story from FanFiction.net and generates pdf at STORY_ID/ folder.
+
+  The following commands are available
+    pull        scrapes story and puts the markdown and template files at STORY_ID/ folder
+    build       compiles *.md files at STORY_ID/ into a single pdf
+    convert     converts STORY_ID/combined.md  to pdf using the template files
+MSG
+      end
+    end
+  end
+end

data/lib/ffnpdf/story.rb

@@ -0,0 +1,218 @@
+require 'httparty'
+require 'nokogiri'
+require 'fileutils'
+require 'open3'
+
+module Ffnpdf
+  class Story
+    FFN_URL = "http://www.fanfiction.net/"
+
+    attr_accessor :custom_url, :error
+
+    def initialize(story_id)
+      @story_id = story_id
+    end
+
+    def check_story_id
+      unless /^[\d]*$/.match @story_id
+        @error = "Story ID is invalid"
+        return false
+      end
+      true
+    end
+
+    def check_story
+      return false unless check_story_id
+
+      test_pull = HTTParty.get(story_url)
+      unless test_pull.code == 200
+        @error = "Story does not exist (#{story_url})"
+        return false
+      end
+
+      unless Nokogiri::HTML(test_pull.body).css(".storytext")
+        @error = "Story does not exist (#{story_url})"
+        return false
+      end
+
+      true
+    end
+
+    def check_story_dir
+      return false unless check_story_id
+      
+      unless File.directory?(@story_id)
+        @error = "Story folder does not exist (#{@story_id}/)"
+        return false
+      end
+
+      unless chapter_mds.count > 0
+        @error = "Story folder does not have markdown files (#{@story_id}/)"
+        return false
+      end
+
+      true
+    end
+
+    def chapter_mds
+      Dir["#{@story_id}/*"].grep(/.md$/).reject { |x| /combined.md$/.match x }.sort
+    end
+
+    def story_url
+      custom_url ? "#{@custom_url}s/#{@story_id}/" : "#{FFN_URL}s/#{@story_id}/"
+    end
+
+    def pull_story
+      return unless check_story
+      FileUtils.mkdir_p @story_id
+      Dir.chdir @story_id
+      
+      tempfile = File.new("temp.html", "w")
+      pull = HTTParty.get(story_url)
+      doc = Nokogiri::HTML(pull.body)
+      doc.css(".storytext")[0].children.each do |paragraph|
+        if /^<(p|hr|i|b)/.match paragraph.to_s
+          tempfile.puts paragraph
+        end
+      end
+      tempfile.close
+
+      Converter.exec 'pandoc temp.html -o 0001.md'
+
+      title = doc.xpath("//div/div/b")[0].text
+      author = doc.xpath("//div/table//a[not(@title or @onclick)]")[0].text
+
+      generate_template(title, author, get_doc_date(doc))
+      generate_variables_file
+      generate_title_page
+
+      chapters = doc.xpath('//form//select[@name="chapter"]/option').count
+      if chapters > 1
+        append_header_to_ch1
+
+        (2..chapters).each do |chapter| 
+          tempfile = File.new("temp.html", "w")
+          tempfile.puts "<h2>Chapter #{chapter}</h2>"
+          pull = HTTParty.get("#{story_url}#{chapter}/")
+          doc = Nokogiri::HTML(pull.body)
+          doc.css(".storytext")[0].children.each do |paragraph|
+            if /^<(p|hr|i|b)/.match paragraph.to_s
+              tempfile.puts paragraph
+            end
+          end
+          tempfile.close
+          Converter.exec("pandoc temp.html -o #{"%04d" % chapter}.md")
+        end
+      end
+      Dir.chdir "../" 
+    end
+
+    def get_doc_date(doc)
+      parsing = doc.xpath('//div[@style="padding-left:1em;padding-right:1em;padding-top:0.5em;"]')[0].text
+      match = /Updated: (.{8})/.match parsing
+      if match
+        return Date.strptime(match[1], "%m-%d-%y").strftime("%d %B %Y")
+      else
+        match = /Published: (.{8})/.match parsing
+        return Date.strptime(match[1], "%m-%d-%y").strftime("%d %B %Y")
+      end
+    end
+
+    def generate_template(title, author, date)
+      template = File.new("xetex.template", "w")
+      contents = <<-CONTENTS
+% Shamelessly ripped off from https://github.com/karlseguin/the-little-mongodb-book
+\\documentclass[$fontsize$,$columns$]{book}
+\\usepackage{fullpage}
+\\usepackage{changepage}
+\\usepackage{fontspec,xltxtra,xunicode}
+\\defaultfontfeatures{Mapping=tex-text,Scale=MatchLowercase}
+\\setmainfont{$mainfont$}
+\\setsansfont{$sansfont$}
+\\setmonofont{$monofont$}
+
+\\setlength{\\parindent}{0pt}
+\\setlength{\\parskip}{12pt plus 2pt minus 1pt}
+\\linespread{1.2}
+
+\\usepackage{listings}
+\\usepackage[dvipsnames,usenames]{color}
+
+$if(fancy-enums)$
+\\usepackage{enumerate}
+$endif$
+\\setcounter{secnumdepth}{-1}
+
+\\usepackage{hyperref}
+\\hypersetup{
+    colorlinks=true,%
+    citecolor=Black,%
+    filecolor=Black,%
+    linkcolor=Black,%
+    urlcolor=Black
+}
+
+\\title{#{title}}
+\\author{#{author}}
+\\date{#{date}}
+\\begin{document}
+$body$
+\\end{document}
+      CONTENTS
+      template.puts(contents)
+      template.close
+    end
+
+    def generate_variables_file
+      variables = File.new("variables.txt", "w")
+      variables.puts("-V paper=a4paper -V hmargin=3cm -V vmargin=3cm -V mainfont=\"DejaVu Serif\" -V sansfont=\"DejaVu Sans\" -V monofont=\"DejaVu Sans Mono\" -V geometry=portrait -V columns=onecolumn -V fontsize=11pt ")
+      variables.close
+    end
+
+    def generate_title_page
+      title_page = File.new("0000.md", "w")
+      contents = <<-CONTENTS
+\\thispagestyle{empty}
+\\maketitle
+\\tableofcontents
+
+      CONTENTS
+      title_page.puts(contents)
+      title_page.close
+    end
+
+    def append_header_to_ch1
+      chapter1 = File.new("0001.temp", "w")
+      chapter1.puts "\#\# Chapter 1"
+      chapter1.puts
+      IO.foreach("0001.md") do |line|
+        chapter1.puts line
+      end
+      chapter1.close
+      FileUtils.rm("0001.md")
+      File.rename("0001.temp", "0001.md")
+    end
+
+    def build_story
+      return unless check_story_dir
+      combine_mds
+      convert_to_pdf
+    end
+
+    def combine_mds
+      combined = File.new("#{@story_id}/combined.md", "w")
+      chapter_mds.each do |chapter|
+        IO.foreach("#{chapter}") do |line|
+          combined.puts(line)
+        end
+        combined.puts("\\clearpage")
+        combined.puts()
+      end
+      combined.close
+    end
+
+    def convert_to_pdf
+      Converter.convert_to_pdf(@story_id)
+    end
+  end
+end

data/lib/ffnpdf/version.rb

@@ -0,0 +1,3 @@
+module Ffnpdf
+  VERSION = "0.0.2"
+end

data/spec/ffnpdf/converter_spec.rb

@@ -0,0 +1,14 @@
+require 'spec_helper'
+
+describe Ffnpdf::Converter do
+  context "exec" do
+    it "should be able to run commands" do
+      Ffnpdf::Converter.exec("pwd")
+    end
+    it "should be able to run syncrhonously" do
+      Ffnpdf::Converter.exec("rm temp.txt")
+      Ffnpdf::Converter.exec("pwd > temp.txt")
+      File.exists?("temp.txt").should == true
+    end
+  end
+end

data/spec/ffnpdf/exec_spec.rb

@@ -0,0 +1,4 @@
+require 'spec_helper'
+
+describe Ffnpdf::Exec do
+end

data/spec/ffnpdf/story_spec.rb

@@ -0,0 +1,138 @@
+require 'spec_helper'
+
+describe Ffnpdf::Story do
+
+  it "should create a story" do
+    story = Ffnpdf::Story.new("1234567")
+    story.custom_url = "http://bryanbibat.github.com/ffnpdf-test/"
+    story.check_story.should == true
+  end
+
+  it "should allow setting custom url" do
+    story = Ffnpdf::Story.new("1234567")
+    story.story_url.should == "http://www.fanfiction.net/s/1234567/"
+    story.custom_url = "http://bryanbibat.github.com/ffnpdf-test/"
+    story.story_url.should == "http://bryanbibat.github.com/ffnpdf-test/s/1234567/"
+  end
+
+  it "should check if story id is valid" do
+    blah = Ffnpdf::Story.new("blah")
+    blah.check_story.should == false
+    blah.error.should == "Story ID is invalid"
+  end
+
+  it "should check story's existence" do
+    story = Ffnpdf::Story.new("0")
+    story.custom_url = "http://bryanbibat.github.com/ffnpdf-test/"
+    story.check_story.should == false
+    story.error.should == "Story does not exist (#{story.story_url})"
+  end
+
+  context "pulling" do
+    before :each do
+      @story = Ffnpdf::Story.new("1234567")
+      @story.custom_url = "http://bryanbibat.github.com/ffnpdf-test/"
+    end
+
+    after :each do
+      if File.directory?("1234567") 
+        FileUtils.rm_rf("1234567")
+      end
+    end
+
+    it "should create a story directory" do
+      pwd = Dir.pwd
+      @story.pull_story
+      Dir.pwd.should == pwd
+      File.should be_directory("1234567")
+      File.should exist("1234567/0001.md")
+      File.should exist("1234567/0000.md")
+      File.should exist("1234567/variables.txt")
+      File.should exist("1234567/xetex.template")
+    end
+  end
+
+  context "building" do
+    after :each do
+      if File.directory?("1234567") 
+        FileUtils.rm_rf("1234567")
+      end
+    end
+
+    it "should check non existence of directory" do
+      story = Ffnpdf::Story.new("1234567")
+      story.custom_url = "http://bryanbibat.github.com/ffnpdf-test/"
+      story.check_story_dir.should == false
+      story.error.should == "Story folder does not exist (1234567/)"
+    end
+
+    it "should check non existence of files" do
+      story = Ffnpdf::Story.new("1234567")
+      story.custom_url = "http://bryanbibat.github.com/ffnpdf-test/"
+      story.pull_story
+      story = Ffnpdf::Story.new("1234567")
+      story.custom_url = "http://bryanbibat.github.com/ffnpdf-test/"
+      FileUtils.rm("1234567/0001.md")
+      FileUtils.rm("1234567/0000.md")
+      story.check_story_dir.should == false
+      story.error.should == "Story folder does not have markdown files (1234567/)"
+    end
+
+    it "should check existence of directory" do
+      story = Ffnpdf::Story.new("1234567")
+      story.custom_url = "http://bryanbibat.github.com/ffnpdf-test/"
+      story.pull_story
+      story = Ffnpdf::Story.new("1234567")
+      story.custom_url = "http://bryanbibat.github.com/ffnpdf-test/"
+      story.check_story_dir.should == true
+    end
+
+  end
+
+  context "pulling multi" do
+    before :each do
+      @story = Ffnpdf::Story.new("0000001")
+      @story.custom_url = "http://bryanbibat.github.com/ffnpdf-test/"
+    end
+
+    after :each do
+      if File.directory?("0000001") 
+        FileUtils.rm_rf("0000001")
+      end
+    end
+
+    it "should create a story directory" do
+      pwd = Dir.pwd
+      @story.pull_story
+      Dir.pwd.should == pwd
+      File.should be_directory("0000001")
+      File.should exist("0000001/0001.md")
+      File.should exist("0000001/0002.md")
+      File.should exist("0000001/0003.md")
+    end
+  end
+
+  context "building multi" do
+    before :each do
+      @story = Ffnpdf::Story.new("0000001")
+      @story.custom_url = "http://bryanbibat.github.com/ffnpdf-test/"
+    end
+
+    after :each do
+      if File.directory?("0000001") 
+        #FileUtils.rm_rf("0000001")
+      end
+    end
+
+    it "should build a story" do
+      pwd = Dir.pwd
+      @story.pull_story
+      Dir.pwd.should == pwd
+      @story.build_story
+      Dir.pwd.should == pwd
+      File.should exist("0000001/combined.md")
+      File.should exist("0000001/combined.pdf")
+    end
+
+  end
+end

data/spec/ffnpdf_spec.rb

@@ -0,0 +1,9 @@
+require 'spec_helper'
+
+describe Ffnpdf do
+  it "has the correct version" do
+    Ffnpdf::VERSION.should == "0.0.1"
+  end
+end
+
+

data/spec/spec_helper.rb

@@ -0,0 +1,4 @@
+require_relative '../lib/ffnpdf'
+require "test_notifier/runner/rspec"
+
+TestNotifier.default_notifier = :notify_send

metadata

@@ -0,0 +1,142 @@
+--- !ruby/object:Gem::Specification
+name: ffnpdf
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+  prerelease: 
+platform: ruby
+authors:
+- Bryan Bibat
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-02-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &79274890 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *79274890
+- !ruby/object:Gem::Dependency
+  name: test_notifier
+  requirement: &79272240 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *79272240
+- !ruby/object:Gem::Dependency
+  name: autotest
+  requirement: &79270640 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *79270640
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: &79269170 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *79269170
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: &79268340 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *79268340
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: &79267380 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *79267380
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: &79266680 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *79266680
+description: Scrapes a story off FF.net, converts it to Markdown, and turns it to
+  PDF with LaTeX
+email:
+- bry@bryanbibat.net
+executables:
+- ffnpdf
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- README.md
+- Rakefile
+- autotest/discover.rb
+- bin/ffnpdf
+- ffnpdf.gemspec
+- lib/ffnpdf.rb
+- lib/ffnpdf/converter.rb
+- lib/ffnpdf/exec.rb
+- lib/ffnpdf/story.rb
+- lib/ffnpdf/version.rb
+- spec/ffnpdf/converter_spec.rb
+- spec/ffnpdf/exec_spec.rb
+- spec/ffnpdf/story_spec.rb
+- spec/ffnpdf_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/bryanbibat/ffnpdf
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: ffnpdf
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: PDF generator for FF.net stories
+test_files: []