elibrum 0.1.0-java

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+gem "nokogiri", "~> 1.5.6"
+gem "erubis", "~> 2.7.0"

data/README.md

@@ -0,0 +1,64 @@
+Elibrum
+=======
+
+Elibrum converts webpages into ebooks. It extracts the text with Boilerpipe and builds the ebook with `ebook-convert`.
+
+JRuby 1.9.2 or greater is required.
+
+## Installation
+
+Install [Calibre](http://calibre-ebook.com/) to get `ebook-convert`.
+
+```
+$ rvm install jruby
+$ rvm use jruby
+$ export JRUBY_OPTS=--1.9
+$ gem install elibrum
+```
+
+## Usage
+
+```
+require "elibrum"
+
+# Must be set. Usually in the following location on OS X:
+Elibrum::EBOOK_CONVERT_PATH = "/Applications/calibre.app/Contents/MacOS/ebook-convert"
+
+Elibrum::Builder.new("git_tutorials", :epub) do
+	add "http://gitready.com/intermediate/2009/01/31/intro-to-rebase.html"
+	add "http://gitready.com/intermediate/2009/02/13/list-remote-branches.html"
+	add "http://gitready.com/advanced/2009/07/31/tig-the-ncurses-front-end-to-git.html"
+end
+
+links = []
+links << "http://techcrunch.com/2013/01/25/eu-enlists-telefonica-cisco-hp-nokia-arm-and-others-to-close-the-700k-it-job-gap-in-europe/"
+links << "http://techcrunch.com/2013/01/25/h265-is-approved/"
+links << "http://techcrunch.com/2013/01/24/nokia-confirms-the-pure-view-was-officially-the-last-symbian-phone/"
+
+Elibrum::Builder.new("blog_posts", [:epub, :mobi, :pdf]) do |b|
+	b.title = "Some Blog Posts"
+	b.author = "TechCrunch"
+
+	b.add *links do |a|
+		a.extractor = :largest_content
+		a.modify do |title, text|
+			title = title.split(" | ").first.strip
+			text = text.split("Comments").last
+			[title, text]
+		end
+	end
+end
+
+# Frontpage of Hacker News as an ebook!
+require "ruby-hackernews"
+Elibrum::Builder.new("frontpage", [:epub, :pdf]) do
+	links = RubyHackernews::Entry.all.map{|e| e.link.href}.reject{|l| l[0...4] != "http"}
+	add *links
+end
+```
+
+## TODO
+
+1. Add tests
+1. Modify Boilerpipe to send user agent and accept a string as input (see Webpage#text)
+1. Use [gae-boilerpipe](https://github.com/gregbayer/gae-boilerpipe) to make the project pure Ruby

data/elibrum.gemspec

@@ -0,0 +1,20 @@
+$:.push File.expand_path("../lib", __FILE__)
+require "elibrum/version"
+
+Gem::Specification.new do |s|
+  s.name          = "elibrum"
+  s.version       = Elibrum::VERSION
+  s.authors       = ["Dave Sescleifer"]
+  s.summary       = "Converts webpages into ebooks"
+  s.description   = "Converts webpages into ebooks."
+  s.email         = "dave@sescleifer.com"
+  s.platform      = "java"
+  s.homepage      = "http://github.com/dsesclei/elibrum"
+  s.require_paths = ["lib"]
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+
+  s.add_runtime_dependency "nokogiri", ["~> 1.5.6"]
+  s.add_runtime_dependency "erubis", ["~> 2.7.0"]
+end

data/lib/elibrum/book.rb

@@ -0,0 +1,20 @@
+module Elibrum
+	class Book
+		attr_accessor :title, :author
+		attr_reader :pages
+
+		def initialize(filename, &block)
+			@title = filename.split(/_|-/).map(&:capitalize).join(" ")
+			@author = "Unknown"
+			@pages = []
+
+			block.arity < 1 ? instance_eval(&block) : block.call(self)
+		end
+
+		def add(*links, &block)
+			links.each do |url|
+				@pages << Webpage.new(url, &block)
+			end
+		end
+	end
+end

data/lib/elibrum/builder.rb

@@ -0,0 +1,40 @@
+module Elibrum
+	class Builder
+		def initialize(filename, formats, template=default_template, &block)
+			raise "Elibrum::EBOOK_CONVERT_PATH is not defined. Put the path to ebook-convert in this constant." unless defined?(EBOOK_CONVERT_PATH)
+			
+			FileUtils.mkdir(".images-temp") unless File.directory?(".images-temp")
+			@book = Book.new(filename, &block)
+			@template = template
+
+			[*formats].each do |format|
+				build(filename, format)
+			end
+
+			FileUtils.rm_rf(".images-temp")
+		end
+
+		def build(filename, format)
+			# Create an HTML file to use as the source file for conversion
+			File.open(".#{filename}-temp.html", "w+") {|f| f.write(html)}
+			# Documentation for ebook-convert can be found here: http://manual.calibre-ebook.com/cli/ebook-convert.html
+			`#{EBOOK_CONVERT_PATH} .#{filename}-temp.html #{filename}.#{format} --title "#{@book.title}" --authors "#{@book.author}" --chapter "//h:pagebreak"`
+			#FileUtils.rm(filename + "-temp.html")
+		end
+
+		private
+
+		def html
+			@html ||= begin
+				template_source = File.read(@template)
+				html = @book.pages.inject("") do |ret, page|
+					ret += Erubis::Eruby.new(template_source).result(title: page.title, text: page.text, url: page.url)
+				end
+			end
+		end
+
+		def default_template
+			File.expand_path("default.erb", File.dirname(__FILE__))
+		end
+	end
+end

data/lib/elibrum/default.erb

@@ -0,0 +1,3 @@
+<pagebreak /> <!-- This is a marker for ebook-convert -->
+<h1><%= title %></h1>
+<span><%= text %></span>

data/lib/elibrum/jars/LICENSE.txt

@@ -0,0 +1,18 @@
+
+ boilerpipe
+
+ Copyright (c) 2009-2011 Christian Kohlschütter
+
+ The author licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ 

data/lib/elibrum/jars/NOTICE.txt

@@ -0,0 +1,24 @@
+
+ boilerpipe
+
+ Copyright (c) 2009-2011 Christian Kohlschütter
+
+ The author licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ 
+ This software contains the following parts which are also provided
+ under the Apache License 2.0 (http://apache.org/licenses/LICENSE-2.0.txt):
+
+    - NekoHTML
+    - Xerces
+

data/lib/elibrum/version.rb

@@ -0,0 +1,3 @@
+module Elibrum
+	VERSION = "0.1.0"
+end

data/lib/elibrum/webpage.rb

@@ -0,0 +1,94 @@
+module Elibrum
+	class Webpage
+		attr_reader :url, :extractor
+
+		def initialize(url, &block)
+			# Delete the trailing / so that URI.join in localize_images doesn't confuse this URL for a directory
+			@url = url[-1] == "/" ? url[0...-1] : url
+			@modify = proc {|title, text| [title, text]}
+			@extractor = CommonExtractors::ARTICLE_EXTRACTOR
+
+			if block
+				block.arity < 1 ? instance_eval(&block) : block.call(self)
+			end
+
+			@title, @text = @modify.call(title, text)
+			@text = localize_images(@text)
+		end
+
+		def title
+			@title ||= Nokogiri::HTML(content).xpath("//title").text
+		end
+
+		def text
+			# The page is loaded both here and in Webpage#content.
+			# TODO: Modify process() to accept a string as input.
+			@text ||= begin
+				highlighter = HTMLHighlighter.newExtractingInstance(true, false)
+				highlighter.process(URL.new(@url), @extractor)
+			rescue Exception => e
+				# Boilerpipe does not pass along a user agent when retrieving sites, so some return a 403
+				msg = "Failed to load #{@url} (#{e})"
+				puts msg
+				msg
+			end
+		end
+
+		def extractor=(extractor_type)
+			@extractor = case extractor_type
+			when :article
+				CommonExtractors::ARTICLE_EXTRACTOR
+			when :canola
+				CommonExtractors::CANOLA_EXTRACTOR
+			when :default
+				CommonExtractors::DEFAULT_EXTRACTOR
+			when :keep_everything
+				CommonExtractors::KEEP_EVERYTHING_EXTRACTOR
+			when :largest_content
+				CommonExtractors::LARGEST_CONTENT_EXTRACTOR
+			else
+				raise "Invalid extractor: #{extractor_type}"
+			end
+		end
+
+		# Allows post-extraction processing of the text.
+		# Useful for removing bits that the extractor accidentally included.
+		def modify(&block)
+			@modify = block
+		end
+
+		private
+
+		def content
+			@content ||= open(@url).read
+		end
+
+		# Download the images to a local folder so that they will appear in the ebook.
+		def localize_images(html)
+			noko = Nokogiri::HTML(html)
+			noko.xpath("//img/@src").each do |url|
+				filetype = url.value.split(".").last.split("?").first
+				filename = ""
+				loop do
+					filename = ".images-temp/" + SecureRandom.hex + "." + filetype
+					break unless File.exists?(filename)
+				end
+
+				# Convert all relative paths to absolute ones so that we can download the images.
+				absolute_url = URI.join(@url, url.value).to_s
+
+				begin
+					url.value = filename
+					File.open(filename, "wb") do |f|
+						img = open(absolute_url).read
+						f.write(img)
+					end
+				rescue Exception => e
+					puts "Failed to load #{absolute_url} (#{e})"
+				end
+			end
+
+			noko.to_html
+		end
+	end
+end

data/lib/elibrum.rb

@@ -0,0 +1,21 @@
+raise "Elibrum only works on JRuby at the moment." unless RUBY_PLATFORM =~ /java/
+
+require "open-uri"
+require "uri"
+require "FileUtils"
+require "SecureRandom"
+require "erubis"
+require "nokogiri"
+require "java"
+require "elibrum/jars/boilerpipe-1.2.0.jar"
+require "elibrum/jars/nekohtml-1.9.13.jar"
+require "elibrum/jars/xerces-2.9.1.jar"
+
+java_import "de.l3s.boilerpipe.extractors.CommonExtractors"
+java_import "de.l3s.boilerpipe.sax.HTMLHighlighter"
+java_import "de.l3s.boilerpipe.sax.HTMLDocument"
+java_import java.net.URL
+
+require "elibrum/builder"
+require "elibrum/book"
+require "elibrum/webpage"

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification
+name: elibrum
+version: !ruby/object:Gem::Version
+  prerelease:
+  version: 0.1.0
+platform: java
+authors:
+- Dave Sescleifer
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2013-01-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.6
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.5.6
+    none: false
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: erubis
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.7.0
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.7.0
+    none: false
+  prerelease: false
+  type: :runtime
+description: Converts webpages into ebooks.
+email: dave@sescleifer.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README.md
+- elibrum.gemspec
+- lib/elibrum.rb
+- lib/elibrum/book.rb
+- lib/elibrum/builder.rb
+- lib/elibrum/default.erb
+- lib/elibrum/jars/LICENSE.txt
+- lib/elibrum/jars/NOTICE.txt
+- lib/elibrum/jars/boilerpipe-1.2.0.jar
+- lib/elibrum/jars/nekohtml-1.9.13.jar
+- lib/elibrum/jars/xerces-2.9.1.jar
+- lib/elibrum/version.rb
+- lib/elibrum/webpage.rb
+homepage: http://github.com/dsesclei/elibrum
+licenses: []
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+  none: false
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+  none: false
+requirements: []
+rubyforge_project:
+rubygems_version: 1.8.24
+signing_key:
+specification_version: 3
+summary: Converts webpages into ebooks
+test_files: []
+...