colread 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a5110d542174b574c2684dcfbeca7cad9fc7ecc6
-  data.tar.gz: 1d2ffdab33b442c885ff8150c3256b08c1dd0f35
+  metadata.gz: eaa58fe8f40e78e6e9e838c227c863bea2fe349f
+  data.tar.gz: 6d25ee9b43562df83027b0beaf301f99f9643b26
 SHA512:
-  metadata.gz: 41f05aff1a11775039a4cd4baea4aee38af87ca5ca71ff95f8b04deb773bfeec6e1396659e4066be2fae0ccebc74c66d2e4ec1a849a44830780a1a1ed0e6dabf
-  data.tar.gz: 80f1e2405b4c100464758550fe5d82105024efcdb9dd0e20e2ab0a9d66f6c8b875ee3fe31175983145cf7bdfa40ef571a0f233d31468fed35689c8b64aae9a3b
+  metadata.gz: e97b37fac6351f7400ed4e0d267f97ba556d0459c407e55ef68e01f55da38928cc44a5b6bcdb3a01d133c38c2b441999664ea786272cec524df566485cc432ab
+  data.tar.gz: 19ebb47c9bdf28e017d880de748123026ffaf71cdb8d068794e0a2792db7e2e3963c0ac93bfccec5a8e811f62e828f45360edf603ec4555fd70da800e748d92d

data/bin/colread

@@ -1,16 +1,15 @@
 #!/usr/bin/env ruby
 
-require 'colread'
+require File.dirname(__FILE__) + '/../lib/colread'
 require 'rubygems'
 require 'optparse'
-#require File.dirname(__FILE__) + '/../lib/readability'
 
-options = { :debug => false, :images => false }
+options = { :encode => 'GBK' }
 options_parser = OptionParser.new do |opts|
-  opts.banner = "Usage: #{File.basename($0)} [options] URL FileName"
+  opts.banner = "Usage: #{File.basename($0)} [options] Url FileName"
 
-  opts.on("-d", "--debug", "Show debug output") do |v|
-    options[:debug] = v
+  opts.on("-e encode", "--encode=encode", "page encode") do |e|
+    options[:encode] = e
   end
 
   opts.on_tail("-h", "--help", "Show this message") do
@@ -20,9 +19,9 @@ options_parser = OptionParser.new do |opts|
 end
 options_parser.parse!
 
-if ARGV.length != 2
+if ARGV.length < 2
   STDERR.puts options_parser
   exit 1
 end
 
-ColRead::Text.new(url: ARGV.first, output: ARGV.last).start
+ColRead::Text.new(url: ARGV.first, output: ARGV.last, encode: options[:encode]).start

data/lib/colread.rb

@@ -15,8 +15,8 @@ end
 module ColRead
   module Core
     def chapters url
-      doc = Nokogiri::HTML(open(url))
-      doc.css('a').group_by{|a| a.indent}.sort_by{|a| a.last.count}.last.last
+      doc = Nokogiri::HTML(_open(url))
+      doc.css('a').select{|a| a.text =~ /\S/ }.group_by{|a| a.indent}.sort_by{|a| a.last.count}.last.last
     end
 
     def ahref a
@@ -31,18 +31,25 @@ module ColRead
 
     def contents chapters
       chapters.each do |a|
-        source = open(ahref(a)).read.encode('utf-8')
+        source = _open(ahref(a))
         content=Nokogiri::HTML(Readability::Document.new(source).content).text
         yield [a.text, content]
       end
     end
+
+    def _open url
+      source = open(url).read
+      source.force_encoding(@encode)
+      source.encode!("utf-8", :undef => :replace, :replace => "?", :invalid => :replace)
+    end
   end
   
   class Text
     include ColRead::Core
-    def initialize(options={url: '',output: ''})
+    def initialize(options={url: '',output: '', encode: ''})
       @url=options[:url]
       @output=options[:output]
+      @encode=options[:encode]
       @root=@url.split(/(?<!\/)\/(?!\/)/).first
     end
     def start

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: colread
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - yunjie chen