arxiv-references 0.1.7.0 → 0.1.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c1fd3a9552e15b293ec4f57b89a2393e840f69d9
-  data.tar.gz: 2cee2a9b1dc53d287c44c79cfd6f18e50d109118
+  metadata.gz: 41b13e1de94b5b60ded925f2621f87e881262b5b
+  data.tar.gz: 70882f9a8f74f0c8c549f23bd9c284fc0b24458e
 SHA512:
-  metadata.gz: d6e24ff8fb896d0658fd7526ec62f61078a377f2b3df8ad200c4bd25cab97b0bba34ca6337991b5806eb048967e7e8ca3d17030020adc46ee8873f03612f963f
-  data.tar.gz: 46c78134b1b051e495162d4ce659e0ea4f7134991eaf824cd2c0b96c2875c4aa0da4ce64db1dbfb87b2db81e97846b7afc3c797f571d54dfd3a7326c66c90c73
+  metadata.gz: 8439088f8eb532a5b9c0f092db5e0c918e2c49929669c6cbe8b8592b32ac88e7b63e515acac7c7cef099f7b51c09e6b7f959dd6247d39b54308805a38f572265
+  data.tar.gz: 9434188bcd8afa713bdc2384bb6c77af5f00ef6c9a06aeabf6f6915e48c35272f5e4f89a8a6159772f052a6487f94642eeb833913817eaec761ee29b3d9119ac

data/lib/arxiv/references/{Arxiv.rb → ArxivApi.rb}

@@ -2,14 +2,14 @@ require 'open-uri'
 require 'nokogiri'
 require 'json'
 
-class Arxiv
+class ArxivApi
   attr_reader :title, :authors, :abstruct, :pdfurl
   attr_accessor :references
   BASE_URL = 'https://arxiv.org'
   def initialize(id)
-    url = "#{BASE_URL}/abs/#{id}" if id.index('http').nil?
+    id = "#{BASE_URL}/abs/#{id}" if id.index('http').nil?
     charset = nil
-    html = open(url) do |f|
+    html = open(id) do |f|
       charset = f.charset
       f.read
     end
@@ -36,6 +36,11 @@ class Arxiv
   def fetch_pdfurl
     "#{BASE_URL}#{@page.xpath('//*[@id="abs"]/div[1]/div[1]/ul/li[1]/a').attr('href').value}"
   end
+
+  def to_json
+    JSON.pretty_generate({title: @title, authors: @authors, abstruct: @abstruct, pdfurl: @pdfurl, references: @references})
+  end
+
 end
 
 

data/lib/arxiv/references/ArxivReferences.rb

@@ -4,7 +4,7 @@ require 'thor'
 require 'pathname'
 lib = Pathname.new(__FILE__).dirname.join().expand_path
 $:.unshift lib.to_s
-require 'myUtil'
+require 'ArxivUtil'
 require 'json'
 
 

data/lib/arxiv/references/ArxivUtil.rb

@@ -0,0 +1,23 @@
+
+$:.unshift Pathname.new(__FILE__).dirname.join().expand_path.to_s
+require 'ArxivApi'
+require 'P3'
+
+
+module ArxivUtil
+  BASE_URL = "https://arxiv.org"
+  def self.fetchFromUrl(urlName, work_dir, use_dir, use_pdf)
+    arxiv = ArxivApi.new(urlName)
+    arxiv.references = P3.fetchFromPdfUrl(arxiv.pdfurl, work_dir, use_dir) if use_pdf || use_pdf.nil? 
+    return arxiv
+  end
+
+  def self.fetchFromArxivId(id, work_dir, use_dir, use_pdf)
+    target_url = "#{BASE_URL}/abs/#{id}" 
+    fetchFromUrl(target_url, work_dir, use_dir, use_pdf)
+  end
+
+  def self.fetchFromPdfUrl(pdfUrl, work_dir, use_dir)
+    return P3.fetchFromPdfUrl(pdfUrl, work_dir, use_dir)
+  end
+end

data/lib/arxiv/references/{myUtil.rb → P3.rb}

@@ -1,16 +1,13 @@
-require 'open-uri'
 require 'digest/sha2'
 require 'time'
 require 'fileutils'
 require 'pty'
 require 'expect'
 require 'pdf-reader'
-require 'nokogiri'
-require 'json'
 
-module ArxivUtil
+class P3
   BASE_URL = "https://arxiv.org"
-  REFERENCE_START_REGEXP = Regexp.new('References|REFERENCES|Reference|REFERENCE')
+  REFERENCE_START_REGEXP = Regexp.new('[rR][eE][fF][eE][rR][eE][nN][cC][eE][sS]*')
   REFERENCE_REGEXP = Regexp.new('(\[[0-9]?[0-9]\]|\[.+?\])')
   def self.makeId
     return Digest::SHA256.hexdigest Time.now.strftime("%F %H:%M:%S")
@@ -44,31 +41,6 @@ module ArxivUtil
     File.delete("#{work_dir}/#{id}-output.pdf")
     File.delete("#{work_dir}/#{id}-output_k2opt.pdf")
   end
-
-
-  def self.fetchFromUrl(urlName, work_dir, use_dir, use_pdf)
-    puts "fetch => #{urlName}"
-    charset = nil
-    html = open(urlName) do |f|
-      charset = f.charset
-      f.read
-    end
-
-    page = Nokogiri::HTML.parse(html, nil, charset)
-    result = {}
-    result[:title] = page.xpath('//*[@id="abs"]/div[2]/h1').children.select{|i| i.name=='text'}.shift.text.gsub(/\n/,'')
-    result[:authors] = page.xpath('//*[@id="abs"]/div[2]/div[2]/a').map(&:text)
-    result[:abstruct] = page.xpath('//*[@id="abs"]/div[2]/blockquote').children.select{|i| i.name = 'text'}.reverse.shift.text
-    result[:pdfurl] = "#{BASE_URL}#{page.xpath('//*[@id="abs"]/div[1]/div[1]/ul/li[1]/a').attr('href').value}"
-    result[:references] = fetchFromPdfUrl(result[:pdfurl], work_dir, use_dir) if use_pdf
-    return result
-  end
-
-  def self.fetchFromArxivId(id, work_dir, use_dir, use_pdf)
-    target_url = "#{BASE_URL}/abs/#{id}" 
-    fetchFromUrl(target_url, work_dir, use_dir, use_pdf)
-  end
-
   def self.fetchPdfFile(pdfUrl,file_name) 
     open(file_name, 'wb') do |o|
       open(pdfUrl) do |data|
@@ -94,7 +66,6 @@ module ArxivUtil
     return getK2Pdf(job_id, work_dir, use_dir)
   end
 
-
   def self.fetchReference(file_name)
     reader = PDF::Reader.new(file_name)
     page_no = reader.
@@ -105,27 +76,25 @@ module ArxivUtil
       map(&:number).
       sort.
       shift
-      puts "Detect References page=> #{page_no} "
+
       ref_page = reader.
         pages.
         select{|i|
           i.number >= page_no
         }.
         map{|i|
-          i.text.gsub(/\n+/,"\n").gsub(/ +/,' ')
+          i.text.gsub(/\n\n+/,"\n").gsub(/ +/,' ').gsub(/-\n +/,'')
         }.
         join(' ').
+        split("\n").
+        join(' ').
         gsub(REFERENCE_REGEXP,"\n\\1").
-        gsub('- ','').
-        split("\n")
-
-        return ref_page[(ref_page.index{|i| i =~ REFERENCE_START_REGEXP}+1)..ref_page.length].
-          select{|i|
-          i.length > 5
-        }
+        split("\n").
+        select{|i| i.length > 15}
+      return ref_page
   end
 
-  def self.fetchFromPdfUrl(pdfUrl, work_dir, use_dir)
+  def self.fetchFromPdfUrl(pdfUrl, work_dir=true, use_dir=true)
     job_id = makeId
     makeDir(job_id, work_dir) if use_dir
     file_name = makeFile(job_id, work_dir, use_dir)

data/lib/arxiv/references/version.rb

@@ -1,5 +1,5 @@
 module Arxiv
   module References
-    VERSION = "0.1.7.0"
+    VERSION = "0.1.7.1"
   end
 end

data/lib/arxiv/references.rb

@@ -4,7 +4,6 @@ lib = Pathname.new(__FILE__).dirname.join().expand_path.to_s
 $:.unshift lib
 
 require "references/version"
-require 'references/myUtil'
 require 'references/ArxivReferences'
 module Arxiv
   module References

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: arxiv-references
 version: !ruby/object:Gem::Version
-  version: 0.1.7.0
+  version: 0.1.7.1
 platform: ruby
 authors:
 - Takahiro Nishimura
@@ -115,10 +115,10 @@ files:
 - bin/setup
 - exe/arxiv-ref
 - lib/arxiv/references.rb
-- lib/arxiv/references/Arxiv.rb
+- lib/arxiv/references/ArxivApi.rb
 - lib/arxiv/references/ArxivReferences.rb
-- lib/arxiv/references/FetchPaperPDF.rb
-- lib/arxiv/references/myUtil.rb
+- lib/arxiv/references/ArxivUtil.rb
+- lib/arxiv/references/P3.rb
 - lib/arxiv/references/version.rb
 homepage: https://github.com/nishimuuu/Arxiv-references
 licenses:

data/lib/arxiv/references/FetchPaperPDF.rb

@@ -1,80 +0,0 @@
-require 'digest/sha2'
-require 'time'
-require 'fileutils'
-require 'pty'
-require 'expect'
-require 'pdf-reader'
-
-class P3
-  def self.fetchPdfFile(pdfUrl,file_name) 
-    open(file_name, 'wb') do |o|
-      open(pdfUrl) do |data|
-        o.write(data.read)
-      end
-    end
-  end
-
-  def self.convertSingleColPdf(job_id, work_dir,file_name, use_dir)
-    cmd = "k2pdfopt -dev kpw #{file_name}"
-    PTY.spawn(cmd) do |i,o|
-      o.sync = true
-      i.expect(/\S.*Enter option above \(h=help, q=quit\):/,10){
-        o.puts "\n"
-        o.flush
-      }
-      while( i.eof? == false )
-        res = i.gets
-        print res
-        break unless res.index('written').nil?
-      end
-    end
-    return getK2Pdf(job_id, work_dir, use_dir)
-  end
-
-  def self.fetchReference(file_name)
-    reader = PDF::Reader.new(file_name)
-    page_no = reader.
-      pages.
-      reject{|i|
-        i.text.index(REFERENCE_START_REGEXP).nil?
-      }.
-      map(&:number).
-      sort.
-      shift
-      puts "Detect References page=> #{page_no} "
-      ref_page = reader.
-        pages.
-        select{|i|
-          i.number >= page_no
-        }.
-        map{|i|
-          i.text.gsub(/\n+/,"\n").gsub(/ +/,' ')
-        }.
-        join(' ').
-        gsub(REFERENCE_REGEXP,"\n\\1").
-        gsub('- ','').
-        split("\n")
-
-        return ref_page[(ref_page.index{|i| i =~ REFERENCE_START_REGEXP}+1)..ref_page.length].
-          select{|i|
-          i.length > 5
-        }
-  end
-
-  def self.fetchFromPdfUrl(pdfUrl, work_dir, use_dir)
-    job_id = makeId
-    makeDir(job_id, work_dir) if use_dir
-    file_name = makeFile(job_id, work_dir, use_dir)
-
-    fetchPdfFile(pdfUrl, file_name)
-    executed_pdf = convertSingleColPdf(job_id, work_dir, file_name, use_dir)
-    references = fetchReference(executed_pdf)
-    if use_dir
-      removeDir(job_id, work_dir) 
-    else
-      removeFile(job_id, work_dir)
-    end
-    return references
-  end
-
-end