arxiv-references 0.1.6.5 → 0.1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2f01c6085fecee426a78a163f1e27d662ac915ab
-  data.tar.gz: e5f51fe5c4eae1a325fdc98527f571710d746fbd
+  metadata.gz: c1fd3a9552e15b293ec4f57b89a2393e840f69d9
+  data.tar.gz: 2cee2a9b1dc53d287c44c79cfd6f18e50d109118
 SHA512:
-  metadata.gz: 916648916d0a88954972d648496ebdbd07b8a20708fb1e266d2e58cb92b27cb87e637d0fbc28198b74581ecbb0f6cd07e71d88110f78fa207dfcb20f06307f0f
-  data.tar.gz: da7e9de22b948832613c042590b4da80c7f3c865125cdea68d614f7c0651c8bae48cfa43c71d2b705e5599e06d42f96004e6dd02be16f2e7af36f6a283321f18
+  metadata.gz: d6e24ff8fb896d0658fd7526ec62f61078a377f2b3df8ad200c4bd25cab97b0bba34ca6337991b5806eb048967e7e8ca3d17030020adc46ee8873f03612f963f
+  data.tar.gz: 46c78134b1b051e495162d4ce659e0ea4f7134991eaf824cd2c0b96c2875c4aa0da4ce64db1dbfb87b2db81e97846b7afc3c797f571d54dfd3a7326c66c90c73

data/README.md

@@ -1,3 +1,4 @@
+[![Build Status](https://travis-ci.org/nishimuuu/Arxiv-references.svg?branch=master)](https://travis-ci.org/nishimuuu/Arxiv-references)
 [![Gem Version](https://badge.fury.io/rb/arxiv-references.svg)](https://badge.fury.io/rb/arxiv-references)
 [![LICENSES](https://img.shields.io/badge/LICENSE-GPL-blue.svg)](https://img.shields.io/badge/LICENSE-GPL-blue.svg)
 [![Code Climate](https://codeclimate.com/github/nishimuuu/Arxiv-references/badges/gpa.svg)](https://codeclimate.com/github/nishimuuu/Arxiv-references)
@@ -6,6 +7,11 @@
 
 Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/arxiv/references`. To experiment with that code, run `bin/console` for an interactive prompt.
 
+## Demo
+[URL](http://153.126.133.121/arxiv-references-api/html/index.html)
+
+[API Document](http://153.126.133.121/arxiv-references-api/html/api.html)
+
 ## Dependencies
 
 - k2pdfopt (http://www.willus.com/k2pdfopt/)
@@ -46,8 +52,8 @@ Or install it yourself as:
 
 ### Options
 --work_dir : [default: /tmp] working directory to convert multi column pdf to one column
---use_dir  : [default: true] create working directory or not
-
+--dir  : [default: true] create working directory or not
+--pdf   : [default: false] if you don't need citations list, add option `--no-pdf`
 
 ## Development
 

data/lib/arxiv/references/Arxiv.rb

@@ -0,0 +1,41 @@
+require 'open-uri'
+require 'nokogiri'
+require 'json'
+
+class Arxiv
+  attr_reader :title, :authors, :abstruct, :pdfurl
+  attr_accessor :references
+  BASE_URL = 'https://arxiv.org'
+  def initialize(id)
+    url = "#{BASE_URL}/abs/#{id}" if id.index('http').nil?
+    charset = nil
+    html = open(url) do |f|
+      charset = f.charset
+      f.read
+    end
+    @page = Nokogiri::HTML.parse(html, nil, charset)
+    @title = fetch_title
+    @authors = fetch_authors
+    @abstruct = fetch_abstruct
+    @pdfurl = fetch_pdfurl
+    @references = nil
+  end
+
+  def fetch_title
+    @page.xpath('//*[@id="abs"]/div[2]/h1').children.select{|i| i.name=='text'}.shift.text.gsub(/\n/,'')
+  end
+
+  def fetch_authors
+    @page.xpath('//*[@id="abs"]/div[2]/div[2]/a').map(&:text)
+  end
+
+  def fetch_abstruct
+    @page.xpath('//*[@id="abs"]/div[2]/blockquote').children.select{|i| i.name = 'text'}.reverse.shift.text
+  end
+
+  def fetch_pdfurl
+    "#{BASE_URL}#{@page.xpath('//*[@id="abs"]/div[1]/div[1]/ul/li[1]/a').attr('href').value}"
+  end
+end
+
+

data/lib/arxiv/references/ArxivReferences.rb

@@ -5,36 +5,30 @@ require 'pathname'
 lib = Pathname.new(__FILE__).dirname.join().expand_path
 $:.unshift lib.to_s
 require 'myUtil'
+require 'json'
 
 
 module ArxivReferences
   class CLI < Thor
     include ArxivUtil
+    class_option 'work_dir', type: :string, aliases: 'Working dir', desc: 'Set working dir(default: /tmp)', default: '/tmp'
+    class_option 'dir', type: :boolean, aliases: 'Working in dir', desc: 'work to make dir or not(default: true)', default: true
+    class_option 'pdf', type: :boolean, aliases: 'Parse PDF', desc: 'fetch pdf information(defaut: true)', default: true
+
+
     desc 'url', 'Extract references from arxiv URL'
-    option 'work_dir', type: :string, aliases: '-work', desc: 'Set working dir(default: /tmp)'
-    option 'use_dir', type: :boolean, aliases: '-use_dir', desc: 'work to make dir or not(default: true)'
     def url(urlName)
-      work_dir = options['work_dir'].nil? ? '/tmp' : options['work_dir']
-      use_dir = options['use_dir'].nil? ? true : options['use_dir']
-      puts ArxivUtil.fetchFromUrl(urlName, work_dir, use_dir)
+      puts ArxivUtil.fetchFromUrl(urlName, options[:work_dir], options[:use_dir], options[:no_pdf]).to_json
     end
 
     desc 'id', 'Extract references from Arxiv id'
-    option 'work_dir', type: :string, aliases: '-work', desc: 'Set working dir(default: /tmp)'
-    option 'use_dir', type: :boolean, aliases: '-use_dir', desc: 'work to make dir or not(default: true)'
     def arxivid(idName)
-      work_dir = options['work_dir'].nil? ? '/tmp' : options['work_dir']
-      use_dir = options['use_dir'].nil? ? true : options['use_dir']
-      puts ArxivUtil.fetchFromArxivId(idName, work_dir, use_dir)
+      puts ArxivUtil.fetchFromArxivId(idName, options[:work_dir], options[:use_dir], options[:no_pdf]).to_json
     end
 
     desc 'pdfurl', 'Extract references from pdf URL'
-    option 'work_dir', type: :string, aliases: '-work', desc: 'Set working dir(default: /tmp)'
-    option 'use_dir', type: :boolean, aliases: '-use_dir', desc: 'work to make dir or not(default: true)'
     def pdfurl(pdfUrlName)
-      work_dir = options['work_dir'].nil? ? '/tmp' : options['work_dir']
-      use_dir = options['use_dir'].nil? ? true : options['use_dir']
-      puts ArxivUtil.fetchFromPdfUrl(pdfUrlName, work_dir, use_dir)
+      puts ArxivUtil.fetchFromPdfUrl(pdfUrlName, options[:work_dir], options[:use_dir]).to_json
     end
   end
 end

data/lib/arxiv/references/FetchPaperPDF.rb

@@ -0,0 +1,80 @@
+require 'digest/sha2'
+require 'time'
+require 'fileutils'
+require 'pty'
+require 'expect'
+require 'pdf-reader'
+
+class P3
+  def self.fetchPdfFile(pdfUrl,file_name) 
+    open(file_name, 'wb') do |o|
+      open(pdfUrl) do |data|
+        o.write(data.read)
+      end
+    end
+  end
+
+  def self.convertSingleColPdf(job_id, work_dir,file_name, use_dir)
+    cmd = "k2pdfopt -dev kpw #{file_name}"
+    PTY.spawn(cmd) do |i,o|
+      o.sync = true
+      i.expect(/\S.*Enter option above \(h=help, q=quit\):/,10){
+        o.puts "\n"
+        o.flush
+      }
+      while( i.eof? == false )
+        res = i.gets
+        print res
+        break unless res.index('written').nil?
+      end
+    end
+    return getK2Pdf(job_id, work_dir, use_dir)
+  end
+
+  def self.fetchReference(file_name)
+    reader = PDF::Reader.new(file_name)
+    page_no = reader.
+      pages.
+      reject{|i|
+        i.text.index(REFERENCE_START_REGEXP).nil?
+      }.
+      map(&:number).
+      sort.
+      shift
+      puts "Detect References page=> #{page_no} "
+      ref_page = reader.
+        pages.
+        select{|i|
+          i.number >= page_no
+        }.
+        map{|i|
+          i.text.gsub(/\n+/,"\n").gsub(/ +/,' ')
+        }.
+        join(' ').
+        gsub(REFERENCE_REGEXP,"\n\\1").
+        gsub('- ','').
+        split("\n")
+
+        return ref_page[(ref_page.index{|i| i =~ REFERENCE_START_REGEXP}+1)..ref_page.length].
+          select{|i|
+          i.length > 5
+        }
+  end
+
+  def self.fetchFromPdfUrl(pdfUrl, work_dir, use_dir)
+    job_id = makeId
+    makeDir(job_id, work_dir) if use_dir
+    file_name = makeFile(job_id, work_dir, use_dir)
+
+    fetchPdfFile(pdfUrl, file_name)
+    executed_pdf = convertSingleColPdf(job_id, work_dir, file_name, use_dir)
+    references = fetchReference(executed_pdf)
+    if use_dir
+      removeDir(job_id, work_dir) 
+    else
+      removeFile(job_id, work_dir)
+    end
+    return references
+  end
+
+end

data/lib/arxiv/references/myUtil.rb

@@ -7,6 +7,7 @@ require 'expect'
 require 'pdf-reader'
 require 'nokogiri'
 require 'json'
+
 module ArxivUtil
   BASE_URL = "https://arxiv.org"
   REFERENCE_START_REGEXP = Regexp.new('References|REFERENCES|Reference|REFERENCE')
@@ -45,7 +46,7 @@ module ArxivUtil
   end
 
 
-  def self.fetchFromUrl(urlName, work_dir, use_dir)
+  def self.fetchFromUrl(urlName, work_dir, use_dir, use_pdf)
     puts "fetch => #{urlName}"
     charset = nil
     html = open(urlName) do |f|
@@ -55,17 +56,17 @@ module ArxivUtil
 
     page = Nokogiri::HTML.parse(html, nil, charset)
     result = {}
-    result[:title] = page.xpath('//*[@id="abs"]/div[2]/h1').text
-    result[:authors] = page.xpath('//*[@id="abs"]/div[2]/div[2]/a').text
-    result[:abstruct] = page.xpath('//*[@id="abs"]/div[2]/blockquote').text
+    result[:title] = page.xpath('//*[@id="abs"]/div[2]/h1').children.select{|i| i.name=='text'}.shift.text.gsub(/\n/,'')
+    result[:authors] = page.xpath('//*[@id="abs"]/div[2]/div[2]/a').map(&:text)
+    result[:abstruct] = page.xpath('//*[@id="abs"]/div[2]/blockquote').children.select{|i| i.name = 'text'}.reverse.shift.text
     result[:pdfurl] = "#{BASE_URL}#{page.xpath('//*[@id="abs"]/div[1]/div[1]/ul/li[1]/a').attr('href').value}"
-    result[:references] = fetchFromPdfUrl(result[:pdfurl], work_dir, use_dir) 
-    return result.to_json
+    result[:references] = fetchFromPdfUrl(result[:pdfurl], work_dir, use_dir) if use_pdf
+    return result
   end
 
-  def self.fetchFromArxivId(id, work_dir, use_dir)
+  def self.fetchFromArxivId(id, work_dir, use_dir, use_pdf)
     target_url = "#{BASE_URL}/abs/#{id}" 
-    fetchFromUrl(target_url, work_dir, use_dir)
+    fetchFromUrl(target_url, work_dir, use_dir, use_pdf)
   end
 
   def self.fetchPdfFile(pdfUrl,file_name) 

data/lib/arxiv/references/version.rb

@@ -1,5 +1,5 @@
 module Arxiv
   module References
-    VERSION = "0.1.6.5"
+    VERSION = "0.1.7.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: arxiv-references
 version: !ruby/object:Gem::Version
-  version: 0.1.6.5
+  version: 0.1.7.0
 platform: ruby
 authors:
 - Takahiro Nishimura
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-05-03 00:00:00.000000000 Z
+date: 2016-05-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -115,7 +115,9 @@ files:
 - bin/setup
 - exe/arxiv-ref
 - lib/arxiv/references.rb
+- lib/arxiv/references/Arxiv.rb
 - lib/arxiv/references/ArxivReferences.rb
+- lib/arxiv/references/FetchPaperPDF.rb
 - lib/arxiv/references/myUtil.rb
 - lib/arxiv/references/version.rb
 homepage: https://github.com/nishimuuu/Arxiv-references