ebook_tools 0.1.1 → 0.1.2

data/CHANGELOG

@@ -1,3 +1,7 @@
+0.1.2 2013.5.27
+  fix bug: 对没有检测到目录结构的书进行错误标注
+  new:  新增异步导入paras
+
 0.1.1 2013.5.26
   fix bug: 提取目录结构时文本内容开始部分存在全角空格而无法正确提取目录结构
   fix bug: 无法提取文本目录中包含“？”等标点符号的目录

data/bin/ebook_tools

@@ -88,6 +88,7 @@ def help(command=nil)
       sed -i 's/^\(\s*[0-9]\{1,3\}\)./\1 /' <file>     : 将 "1.xxx" 格式替换成 "1 xxx"
       iconv --from-code GB2312 -t UTF-8 -c -o <output file> <file> : 将GB2312格式文件转换成UTF-8
       sed -i 's/\s*第.\{1,3\}章.*[0-9]\{1,3\}$//g' <file>  : 将 "第xxx章 xxxx1" 格式清除。主要用于有些pdf文档转换过来时的页眉信息。
+      sed -i 's/第\([0-9]\+\)节：//g'  <file>   : 目录前面有第*节时需要清除
     EOF
   end
 

data/bin/para_import_scheduling

@@ -0,0 +1,73 @@
+#!/usr/bin/env ruby
+# encoding: UTF-8
+require 'rubygems'
+require 'optparse'
+require File.join(File.expand_path('../../',__FILE__),'lib','utils')
+require File.join(File.expand_path('../../',__FILE__),'workers','para_import_worker')
+
+def help
+  puts <<-EOF
+    usage:
+      para_import_scheduling [options] <docbook>
+
+    docbook: 已经标注重点段落的书文件或目录
+
+    options:
+      -H <host> , --host <host> : mongodb服务器,默认为localhost
+      -P <port> , --port <port> : mongodb服务器端口号,默认为27017 (Mongo默认端口号)
+      -D <database>, --database <database>  : 重点段落要存放的数据库
+      -C <collection>, --collection <collection> : 重点段落存放的集合 
+  EOF
+  exit
+end
+
+
+options = {:host=>'localhost',:port=>27017,:database=>'resource_development',:collection=>'paras'}
+opts = OptionParser.new do |opts|
+  opts.on('-H host','--host host') do |host|
+    options[:host] = host
+  end
+
+  opts.on('-P port','--port port') do |port|
+    options[:port] = port.to_i
+  end
+
+  opts.on('-D database','--database database') do |database|
+    options[:database] = database
+  end
+
+  opts.on('-C collection','--collection collection') do |collection|
+    options[:collection] = collection
+  end
+
+  opts.on('-h','--help') do
+    help
+  end
+end
+opts.parse ARGV
+
+docbook = ARGV[-1]
+
+if docbook.nil?
+  help
+end
+
+unless File.exists?(docbook)
+  raise "错误:指定的docbook不存在。"
+end
+
+files = if File.directory?(docbook)
+  Utils.scan_file_from_dir(docbook,{:format=>'.xml'})
+else
+  [docbook]
+end
+
+files.each do |file|
+  begin
+    #ParasInMongo.file_in_mongo(file,options)
+    ParaImportWorker.perform_async(file,options)
+  rescue
+    puts "error: #{file} to  job failure!"
+  end
+end
+puts "success: #{docbook} to job successfully!"

data/ebook_tools.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{ebook_tools}
-  s.version = '0.1.1'
+  s.version = '0.1.2'
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Aaron"]
@@ -23,6 +23,7 @@ Gem::Specification.new do |s|
     "bin/ebook_tools",
     "bin/para_import_mongo",
     "bin/doc_book_import_mongo",
+    "bin/para_import_scheduling",
     "bin/xml2json",
     "lib/ebook_tools.rb",
     "lib/txt_book.rb",
@@ -33,6 +34,7 @@ Gem::Specification.new do |s|
     "lib/utils.rb",
     "lib/paras_in_mongo.rb",
     "lib/doc_book_in_mongo.rb",
+    "workers/para_import_worker.rb",
     "ebook_tools.gemspec"
   ]
   s.add_dependency(%q<uuid>)
@@ -43,4 +45,5 @@ Gem::Specification.new do |s|
   s.add_dependency(%q<nokogiri>)
   s.add_dependency(%q<levenshtein>)
   s.add_dependency(%q<moped>)
+  s.add_dependency(%q<sidekiq>)
 end

data/lib/ebook_tools.rb

@@ -246,8 +246,10 @@ module EbookTools
           if extract_book_struct_to_file(file,dest_file)
             puts "success: extract book struct  successfully!"
           else
+            new_file = File.join(File.dirname(file),"[err]#{basename}#{extname}")
+            FileUtils.mv(file,new_file,:force=>true)
             puts "警告: 没有检测到书结构信息."
-          end            
+          end
         rescue Exception => e
           puts "error: #{file} \n#{e.backtrace.join("\n")}"
         end

data/lib/paras_in_mongo.rb

@@ -1,13 +1,15 @@
 # encoding: UTF-8
 require 'moped'
 require 'nokogiri'
+require 'active_support'
 
 module ParasInMongo
   extend self
 
   def file_in_mongo(filename,options={})
-    session = Moped::Session.new([ "#{options[:host]}:#{options[:port]}" ])
-    session.use options[:database]
+    options = options.stringify_keys
+    session = Moped::Session.new([ "#{options['host']}:#{options['port']}" ])
+    session.use options['database']
 
     doc = Nokogiri::XML(File.open(filename).read)
 
@@ -33,7 +35,7 @@ module ParasInMongo
       para_attrs = para_attrs.merge(keywords: keywords, content: content)
       section = para.parent.search("info title").text
       para_attrs = para_attrs.merge(source: source.merge(location: {section: section}))
-      session[options[:collection]].insert(para_attrs)
+      session[options['collection']].insert(para_attrs)
     end 
   end
 end

data/lib/utils.rb

@@ -195,7 +195,7 @@ module Utils
   def scan_file_from_dir(dir,options={})
     files = []
     walk_dir(dir,options) do |file|
-      files << file.to_s
+      files << file.realpath.to_s
     end
     files
   end

data/workers/para_import_worker.rb

@@ -0,0 +1,22 @@
+require 'sidekiq'
+require File.join(File.expand_path('../../',__FILE__),'lib','paras_in_mongo')
+
+# If your client is single-threaded, we just need a single connection in our Redis connection pool
+Sidekiq.configure_client do |config|
+  config.redis = { :namespace => 'x', :size => 1, :url => 'redis://localhost:6379/14' }
+end
+
+# Sidekiq server is multi-threaded so our Redis connection pool size defaults to concurrency (-c)
+Sidekiq.configure_server do |config|
+  config.redis = { :namespace => 'x', :url => 'redis://localhost:6379/14' }
+end
+
+class ParaImportWorker
+  include Sidekiq::Worker
+
+  def perform(filename,options={})
+    puts "Workin' #{filename}"
+    ParasInMongo.file_in_mongo(filename,options)
+    puts "#{filename} in mongo successfully"
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ebook_tools
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
   prerelease: 
 platform: ruby
 authors:
@@ -139,6 +139,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 电子书工具集.
 email: aaron@nonobo.com
 executables:
@@ -153,6 +169,7 @@ files:
 - bin/ebook_tools
 - bin/para_import_mongo
 - bin/doc_book_import_mongo
+- bin/para_import_scheduling
 - bin/xml2json
 - lib/ebook_tools.rb
 - lib/txt_book.rb
@@ -163,6 +180,7 @@ files:
 - lib/utils.rb
 - lib/paras_in_mongo.rb
 - lib/doc_book_in_mongo.rb
+- workers/para_import_worker.rb
 - ebook_tools.gemspec
 homepage: 
 licenses: []