textminer 0.1.0 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5e954930af35bca6c9752b9f9d660eb675cd4bea
-  data.tar.gz: c451255b116b2eae5a52d66adc3619e39c7f10c4
+  metadata.gz: c6c80a22022bb38bc141dc50e8da5d913db03946
+  data.tar.gz: 957cf24214f95f1b2d8309f2fd1a2e2aa7b6ca69
 SHA512:
-  metadata.gz: 0ff5aacaf4be3b3a797f6cc8435c9c4a2bec7be98d695986f2b27a044b98492f5729cd58979952dacf96777d2306c2c8b9d9abda8dc7bfa94e9080a4f4ae8f6c
-  data.tar.gz: 64a5fd5ebb268403c12350d444794efde329e97fb14e7ad47959d4af5aa3465306f135174e183b4ce11ab1009ba24fc29ebc47d7b605e90a4e4e308274264671
+  metadata.gz: 9837bd893866ef35e420d928bf02f3151783b345d39f758ed5ddce8b98c6df92147ff518b889d5cab33f84aa62ad795a3e7e1e2c6ad18cfd7a9a3060589293eb
+  data.tar.gz: 1151759369e8007f85ad73f24872f409ffcb70e99ad114e7a48e623c48a53ea118c7ed13a4ded171fb64823a2e21e342000f77766aaac93b12493335ace58f1d

data/.travis.yml

@@ -1,5 +1,4 @@
 language: ruby
 rvm:
-  - 1.9.3
   - 2.1.7
   - 2.2.3

data/CHANGELOG.md

@@ -0,0 +1,9 @@
+## 0.1.5 (2015-12-04)
+
+* Now using `serrano` gem for interacting with the Crossref API
+* Changed `links` method to `search`
+* Changed `fetch` method to accept a URL for a full text article instead of a DOI
+
+## 0.1.0 (2015-08-24)
+
+* First version to Rubygems

data/Gemfile.lock

@@ -1,12 +1,18 @@
 PATH
   remote: .
   specs:
-    textminer (0.1.0)
+    textminer (0.1.5)
+      faraday (~> 0.9.1)
+      faraday_middleware (~> 0.10.0)
       httparty (~> 0.13)
       json (~> 1.8)
-      launchy (~> 2.4, >= 2.4.2)
+      launchy (~> 2.4, >= 2.4.3)
+      multi_json (~> 1.0)
+      nokogiri (~> 1.6, >= 1.6.6.2)
       pdf-reader (~> 1.3)
+      serrano (~> 0.1.4.1)
       thor (~> 0.19)
+      uuidtools (~> 2.1, >= 2.1.5)
 
 GEM
   remote: https://rubygems.org/
@@ -21,14 +27,23 @@ GEM
       simplecov
       url
     docile (1.1.5)
+    faraday (0.9.1)
+      multipart-post (>= 1.2, < 3)
+    faraday_middleware (0.10.0)
+      faraday (>= 0.7.4, < 0.10)
     hashery (2.1.1)
-    httparty (0.13.5)
+    httparty (0.13.7)
       json (~> 1.8)
       multi_xml (>= 0.5.2)
     json (1.8.3)
     launchy (2.4.3)
       addressable (~> 2.3)
+    mini_portile (0.6.2)
+    multi_json (1.11.2)
     multi_xml (0.5.5)
+    multipart-post (2.0.0)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
     oga (1.2.3)
       ast
       ruby-ll (~> 2.1)
@@ -44,6 +59,11 @@ GEM
       ansi
       ast
     ruby-rc4 (0.1.5)
+    serrano (0.1.4.1)
+      faraday (~> 0.9.1)
+      faraday_middleware (~> 0.10.0)
+      multi_json (~> 1.0)
+      thor (~> 0.19)
     simplecov (0.10.0)
       docile (~> 1.1.0)
       json (~> 1.8)
@@ -54,6 +74,7 @@ GEM
     thor (0.19.1)
     ttfunk (1.4.0)
     url (0.3.2)
+    uuidtools (2.1.5)
 
 PLATFORMS
   ruby
@@ -66,3 +87,6 @@ DEPENDENCIES
   simplecov (~> 0.10)
   test-unit (~> 3.1)
   textminer!
+
+BUNDLED WITH
+   1.10.6

data/README.md

@@ -1,24 +1,29 @@
 textminer
 =========
 
-[![Build Status](https://api.travis-ci.org/sckott/textminer.png)](https://travis-ci.org/sckott/textminer)
+[![gem version](https://img.shields.io/gem/v/textminer.svg)](https://rubygems.org/gems/textminer)
+[![Build Status](https://travis-ci.org/sckott/textminer.svg?branch=master)](https://travis-ci.org/sckott/textminer)
 [![codecov.io](http://codecov.io/github/sckott/textminer/coverage.svg?branch=master)](http://codecov.io/github/sckott/textminer?branch=master)
 
-__This is alpha software, so expect changes__
-
-## What is it?
-
 __`textminer` helps you text mine through Crossref's TDM (Text & Data Mining) services:__
 
 ## Changes
 
-For changes see the [NEWS file](https://github.com/sckott/textminer/blob/master/NEWS.md).
+For changes see the [CHANGELOG][changelog]
+
+## gem API
+
+* `Textiner.search` - search by DOI, query string, filters, etc. to get Crossref metadata, which you can use downstream to get full text links. This method essentially wraps `Serrano.works()`, but only a subset of params - this interface may change depending on feedback.
+* `Textiner.fetch` - Fetch full text given a url, supports Crossref's Text and Data Mining service
+* `Textiner.extract` - Extract text from a pdf
 
 ## Install
 
 ### Release version
 
-Not on RubyGems yet
+```
+gem install textminer
+```
 
 ### Development version
 
@@ -28,89 +33,87 @@ cd textminer
 rake install
 ```
 
-## Within Ruby
+## Examples
+
+### Within Ruby
+
+#### Search
 
 Search by DOI
 
 ```ruby
 require 'textminer'
-out = textminer.links("10.5555/515151")
+# link to full text available
+Textminer.search(doi: '10.7554/elife.06430')
+# no link to full text available
+Textminer.search(doi: "10.1371/journal.pone.0000308")
 ```
 
-Get the pdf link
-
-```ruby
-out.pdf
-```
+Many DOIs at once
 
 ```ruby
-"http://annalsofpsychoceramics.labs.crossref.org/fulltext/10.5555/515151.pdf"
+require 'serrano'
+dois = Serrano.random_dois(sample: 6)
+Textminer.search(doi: dois)
 ```
 
-Get the xml link
+Search with filters
 
 ```ruby
-out.xml
+Textminer.search(filter: {has_full_text: true})
 ```
 
-```ruby
-"http://annalsofpsychoceramics.labs.crossref.org/fulltext/10.5555/515151.xml"
-```
+#### Get full text links
 
-Fetch XML
+The object returned form `Textminer.search` is a class, which has methods for pulling out all links, xml only, pdf only, or plain text only
 
 ```ruby
-Textminer.fetch("10.3897/phytokeys.42.7604", "xml")
+x = Textminer.search(filter: {has_full_text: true})
+x.links_xml
+x.links_pdf
+x.links_plain
 ```
 
-```ruby
-=> {"article"=>
-  {"front"=>
-    {"journal_meta"=>
-      {"journal_id"=>
-        {"__content__"=>"PhytoKeys", "journal_id_type"=>"publisher-id"},
-       "journal_title_group"=>
-        {"journal_title"=>{"__content__"=>"PhytoKeys", "lang"=>"en"},
-         "abbrev_journal_title"=>{"__content__"=>"PhytoKeys", "lang"=>"en"}},
-       "issn"=>
-        [{"__content__"=>"1314-2011", "pub_type"=>"ppub"},
-         {"__content__"=>"1314-2003", "pub_type"=>"epub"}],
-       "publisher"=>{"publisher_name"=>"Pensoft Publishers"}},
-     "article_meta"=>
-
-...
-```
+#### Fetch full text
 
-Fetch PDF
+`Textminer.fetch()` gets full text based on URL input. We determine how to pull down and parse the content based on content type.
 
 ```ruby
-Textminer.fetch("10.3897/phytokeys.42.7604", "pdf")
+# get some metadata
+res = Textminer.search(member: 2258, filter: {has_full_text: true});
+# get links
+links = res.links_xml(true);
+# Get full text for an article
+res = Textminer.fetch(url: links[0]);
+# url
+res.url
+# file path
+res.path
+# content type
+res.type
+# parse content
+res.parse
 ```
 
-> pdf written to disk
-
-## On the CLI
+#### Extract text from PDF
 
-Get links
+`Textminer.extract()` extracts text from a pdf, given a path for a pdf
 
-```sh
-tm links 10.3897/phytokeys.42.7604
+```ruby
+res = Textminer.search(member: 2258, filter: {has_full_text: true});
+links = res.links_pdf(true);
+res = Textminer.fetch(url: links[0]);
+Textminer.extract(res.path)
 ```
 
-```sh
-http://phytokeys.pensoft.net/lib/ajax_srv/article_elements_srv.php?action=download_xml&item_id=4190
-http://phytokeys.pensoft.net/lib/ajax_srv/article_elements_srv.php?action=download_pdf&item_id=4190
-```
+### On the CLI
 
-More than one DOI:
-
-```sh
-tm links '10.3897/phytokeys.42.7604,10.3897/zookeys.516.9439'
-```
+Coming soon...
 
 ## To do
 
 * CLI executable
-* get actual full text
 * better test suite
-* documentation
+* better documentation
+
+[changelog]: https://github.com/sckott/textminer/blob/master/CHANGELOG.md

data/Rakefile

@@ -3,20 +3,35 @@ require 'rake/testtask'
 
 Rake::TestTask.new do |t|
   t.libs << "test"
-  t.test_files = FileList['test/test*.rb']
+  t.test_files = FileList['test/test-*.rb']
   t.verbose = true
 end
 
 desc "Run tests"
 task :default => :test
 
+desc "Build textminer docs"
+task :docs do
+	system "yardoc"
+end
+
+desc "bundle install"
+task :bundle do
+  system "bundle install"
+end
+
+desc "clean out builds"
+task :clean do
+  system "ls | grep [0-9].gem | xargs rm"
+end
+
 desc "Build textminer"
 task :build do
 	system "gem build textminer.gemspec"
 end
 
 desc "Install textminer"
-task :install => :build do
+task :install => [:bundle, :build] do
 	system "gem install textminer-#{Textminer::VERSION}.gem"
 end
 

data/bin/tm

@@ -14,7 +14,7 @@ class Tm < Thor
   def links(tt)
   	tt = "#{tt}"
     tt = tt.to_s.split(',')
-    out = Textminer.links(tt).all
+    out = Textminer.search(doi: tt).links(true)
     puts out
   end
 end

data/extra/fetch_method.rb

@@ -0,0 +1,17 @@
+##
+# Thin layer around pdf-reader gem's PDF::Reader
+#
+# @param doi [Array] A DOI, digital object identifier
+# @param type [Array] One of two options to download: xml (default) or pdf
+#
+# @example
+#     require 'textminer'
+#     # fetch full text by DOI - xml by default
+#     Textminer.fetch("10.3897/phytokeys.42.7604")
+#     # many DOIs - xml output
+#     res = Textminer.fetch(["10.3897/phytokeys.42.7604", "10.3897/zookeys.516.9439"])
+#     # fetch full text - pdf
+#     Textminer.fetch("10.3897/phytokeys.42.7604", "pdf")
+def self.fetch(doi, type = 'xml')
+  Fetch.new(doi, type).fetchtext
+end

data/lib/textminer.rb

@@ -1,49 +1,124 @@
 require 'httparty'
 require 'json'
 require 'pdf-reader'
+require 'serrano'
+require "textminer/miner"
 require "textminer/version"
 require "textminer/request"
 require "textminer/response"
-require "textminer/fetch"
 
 module Textminer
+  extend Configuration
+
+  define_setting :tdm_key
+
   ##
-  # Get links meant for text mining
+  # Search for papers and get full text links
   #
   # @param doi [Array] A DOI, digital object identifier
+  # @param options [Array] Curl request options
   # @return [Array] the output
   #
   # @example
   #     require 'textminer'
   #     # link to full text available
-  #     Textminer.links("10.5555/515151")
+  #     Textminer.search(doi: '10.3897/phytokeys.42.7604')
   #     # no link to full text available
-  #     Textminer.links("10.1371/journal.pone.0000308")
+  #     Textminer.search(doi: "10.1371/journal.pone.0000308")
   #     # many DOIs at once
-  #     res = Textminer.links(["10.3897/phytokeys.42.7604", "10.3897/zookeys.516.9439"])
+  #     require 'serrano'
+  #     dois = Serrano.random_dois(sample: 6)
+  #     res = Textminer.search(doi: dois)
+  #     res = Textminer.search(doi: ["10.3897/phytokeys.42.7604", "10.3897/zookeys.516.9439"])
   #     res.links
-  #     res.pdf
-  #     res.xml
-  def self.links(doi)
-    Request.new(doi).perform
+  #     res.links_pdf
+  #     res.links_xml
+  #     res.links_plain
+  #     # only full text available
+  #     x = Textminer.search(doi: '10.3816/clm.2001.n.006')
+  #     x.links_xml
+  #     x.links_plain
+  #     x.links_pdf
+  #     # no dois
+  #     x = Textminer.search(filter: {has_full_text: true})
+  #     x.links_xml
+  #     x.links_plain
+  #     x = Textminer.search(member: 311, filter: {has_full_text: true})
+  #     x.links_pdf
+  def self.search(doi: nil, member: nil, filter: nil, limit: nil, options: nil)
+    Request.new(doi, member, filter, limit, options).perform
   end
 
   ##
-  # Thin layer around pdf-reader gem's PDF::Reader
+  # Get full text
   #
-  # @param doi [Array] A DOI, digital object identifier
-  # @param type [Array] One of two options to download: xml (default) or pdf
+  # Work easily for open access papers, but for closed. For non-OA papers, use
+  # Crossref's Text and Data Mining service, which requires authentication and
+  # pre-authorized IP address. Go to https://apps.crossref.org/clickthrough/researchers
+  # to sign up for the TDM service, to get your key. The only publishers
+  # taking part at this time are Elsevier and Wiley.
+  #
+  # @param url [String] A url for full text
+  # @return [Mined] An object of class Mined, with methods for extracting
+  # the url requested, the file path, and parsing the plain text, XML, or extracting
+  # text from the pdf.
   #
   # @example
-  #     require 'textminer'
-  #     # fetch full text by DOI - xml by default
-  #     Textminer.fetch("10.3897/phytokeys.42.7604")
-  #     # many DOIs - xml output
-  #     res = Textminer.fetch(["10.3897/phytokeys.42.7604", "10.3897/zookeys.516.9439"])
-  #     # fetch full text - pdf
-  #     Textminer.fetch("10.3897/phytokeys.42.7604", "pdf")
-  def self.fetch(doi, type = 'xml')
-    Fetch.new(doi, type).fetchtext
+  #   require 'textminer'
+  #   # Set authorization
+  #   Textminer.configuration do |config|
+  #     config.tdm_key = "<your key>"
+  #   end
+  #   # Get some elsevier works
+  #   res = Textminer.search(member: 78, filter: {has_full_text: true});
+  #   links = res.links_xml(true);
+  #   # Get full text for an article
+  #   out = Textminer.fetch(url: links[0]);
+  #   out.url
+  #   out.path
+  #   out.type
+  #   xml = out.parse()
+  #   puts xml
+  #   xml.xpath('//xocs:cover-date-text', xml.root.namespaces).text
+  #   # Get lots of articles
+  #   links = links[1..3]
+  #   out = links.collect{ |x| Textminer.fetch(url: x) }
+  #   out.collect{ |z| z.path }
+  #   out.collect{ |z| z.parse }
+  #   zz = out[0].parse
+  #   zz.xpath('//xocs:cover-date-text', zz.root.namespaces).text
+  #
+  #   ## plain text
+  #   # get full text links, here doing xml
+  #   links = res.links_plain(true);
+  #   # Get full text for an article
+  #   res = Textminer.fetch(url: links[0]);
+  #   res.url
+  #   res.parse
+  #
+  #   # With open access content - using Pensoft
+  #   res = Textminer.search(member: 2258, filter: {has_full_text: true});
+  #   links = res.links_xml(true);
+  #   # Get full text for an article
+  #   res = Textminer.fetch(url: links[0]);
+  #   res.url
+  #   res.parse
+  #
+  #   # OA content - pdfs, using pensoft again
+  #   res = Textminer.search(member: 2258, filter: {has_full_text: true});
+  #   links = res.links_pdf(true);
+  #   # Get full text for an article
+  #   res = Textminer.fetch(url: links[0]);
+  #   # url used
+  #   res.url
+  #   # document type
+  #   res.type
+  #   # document path on your machine
+  #   res.path
+  #   # get text
+  #   res.parse
+  def self.fetch(url)
+    Miner.new(url).perform
   end
 
   ##
@@ -52,15 +127,34 @@ module Textminer
   # @param path [String] Path to a pdf file downloaded via {fetch}, or
   #   another way.
   #
+  # This method is used internally within fetch to parse PDFs.
+  #
   # @example
-  #     require 'textminer'
-  #     # fetch full text - pdf
-  #     res = Textminer.fetch("10.3897/phytokeys.42.7604", "pdf")
-  #     # extract pdf to text
-  #     Textminer.extract(res)
+  #   require 'textminer'
+  #   res = Textminer.search(member: 2258, filter: {has_full_text: true});
+  #   links = res.links_pdf(true);
+  #   # Get full text for an article
+  #   out = Textminer.fetch(url: links[0]);
+  #   # extract pdf to text
+  #   Textminer.extract(out.path)
   def self.extract(path)
     rr = PDF::Reader.new(path)
     rr.pages.map { |page| page.text }.join("\n")
   end
 
+  protected
+
+  def self.link_switch(x, y)
+    case y
+    when nil
+      x.links
+    when 'xml'
+      x.links_xml
+    when 'pdf'
+      x.links_pdf
+    when 'plain'
+      x.links_plain
+    end
+  end
+
 end

data/lib/textminer/helpers/configuration.rb

@@ -0,0 +1,26 @@
+# taken from: https://viget.com/extend/easy-gem-configuration-variables-with-defaults
+module Configuration
+
+  def configuration
+    yield self
+  end
+
+  def define_setting(name, default = nil)
+    class_variable_set("@@#{name}", default)
+    define_class_method "#{name}=" do |value|
+      class_variable_set("@@#{name}", value)
+    end
+    define_class_method name do
+      class_variable_get("@@#{name}")
+    end
+  end
+
+  private
+
+  def define_class_method(name, &block)
+    (class << self; self; end).instance_eval do
+      define_method name, &block
+    end
+  end
+
+end

data/lib/textminer/link_methods_array.rb

@@ -0,0 +1,54 @@
+# Array methods
+class Array
+  def links(just_urls = true)
+    return self.collect{ |x| x.links(just_urls) }.flatten
+    # if temp.length == 1
+    #   return tmp[0]
+    # else
+    #   return tmp
+    # end
+  	# tmp = self.collect{ |x| x['message']['link'] }
+  	# return parse_link(tmp, just_urls)
+  end
+end
+
+class Array
+  def links_xml(just_urls = true)
+    self.collect { |z| z.links_xml(just_urls) }[0]
+    # return parse_link(self.collect { |z| z.links_xml }[0], just_urls)
+  	# return parse_link(pull_link(self, '^application\/xml$|^text\/xml$'), just_urls)
+  end
+end
+
+class Array
+  def links_pdf(just_urls = true)
+    self.collect { |z| z.links_pdf(just_urls) }[0]
+    # return parse_link(self.collect { |z| z.links_pdf }[0], just_urls)
+    # return parse_link(pull_link(self, '^application\/pdf$'), just_urls)
+  end
+end
+
+class Array
+  def links_plain(just_urls = true)
+    self.collect { |z| z.links_plain(just_urls) }[0]
+    # return parse_link(self.collect { |z| z.links_plain }[0], just_urls)
+    # return parse_link(pull_link(self, '^application\/plain$|^text\/plain$'), just_urls)
+  end
+end
+
+# def pull_link(x, y)
+#   return x.collect { |z| z.links_xml }[0]
+#   # return x.collect { |z| z['message']['link'] }.compact.collect { |z| z.compact.select { |w| w['content-type'].match(/#{y}/) } }
+# end
+
+# def parse_link(x, just_urls)
+# 	if x.nil?
+# 		return x
+# 	else
+#   	if just_urls
+#   		return x.compact.collect { |z| z.collect{ |y| y['URL'] }}.flatten
+#   	else
+#   		return x
+#   	end
+#   end
+# end

data/lib/textminer/link_methods_hash.rb

@@ -0,0 +1,71 @@
+# Hash methods
+class Hash
+  def links(just_urls = true)
+    if self['message']['items'].nil?
+      tmp = self['message']['link']
+      if tmp.nil?
+        tmp = nil
+      else
+        tmp = tmp.reject { |c| c.empty? }
+      end
+    else
+      tmp = self['message']['items'].collect { |x| x['link'] }.reject { |c| c.empty? }
+    end
+
+  	return parse_links(tmp, just_urls)
+  end
+end
+
+class Hash
+  def links_xml(just_urls = true)
+  	return parse_links(pull_links(self, '^application\/xml$|^text\/xml$'), just_urls)
+  end
+end
+
+class Hash
+  def links_pdf(just_urls = true)
+  	return parse_links(pull_links(self, '^application\/pdf$'), just_urls)
+  end
+end
+
+class Hash
+  def links_plain(just_urls = true)
+  	return parse_links(pull_links(self, '^application\/plain$|^text\/plain$'), just_urls)
+  end
+end
+
+def pull_links(x, y)
+  if x['message']['items'].nil?
+    tmp = self['message']['link']
+    if tmp.nil?
+      return nil
+    else
+      return tmp.select { |z| z['content-type'].match(/#{y}/) }.reject { |c| c.empty? }
+    end
+  else
+    return x['message']['items'].collect { |x| x['link'].select { |z| z['content-type'].match(/#{y}/) } }.reject { |c| c.empty? }
+  end
+end
+
+def parse_links(x, just_urls)
+  if x.nil?
+    return nil
+  else
+  	if x.empty?
+  		return x
+  	else
+    	if just_urls
+        if x[0].class != Array
+          # return x[0]['URL']
+          return x.collect { |x| x['URL'] }.flatten
+        else
+          return x.collect { |x| x.collect { |z| z['URL'] }}.flatten
+          # return x.collect { |x| x['URL'] }.flatten.compact
+      		# return x.collect { |x| x.collect { |z| z['URL'] }}.flatten
+        end
+    	else
+    		return x
+    	end
+    end
+  end
+end

data/lib/textminer/mine_utils.rb

@@ -0,0 +1,65 @@
+require 'nokogiri'
+require 'uuidtools'
+
+def detect_type(x)
+  ctype = x.headers['content-type']
+  case ctype
+  when 'text/xml'
+    'xml'
+  when 'text/plain'
+    'plain'
+  when 'application/pdf'
+    'pdf'
+  end
+end
+
+def make_ext(x)
+  case x
+  when 'xml'
+    'xml'
+  when 'plain'
+    'txt'
+  when 'pdf'
+    'pdf'
+  end
+end
+
+def make_path(type)
+  # id = x.split('article/')[1].split('?')[0]
+  # path = id + '.' + type
+  # return path
+  type = make_ext(type)
+  uuid = UUIDTools::UUID.random_create.to_s
+  path = uuid + '.' + type
+  return path
+end
+
+def write_disk(res, path)
+  f = File.new(path, "wb")
+  f.write(res.body)
+  f.close()
+end
+
+def read_disk(path)
+  return File.read(path)
+end
+
+def parse_xml(x)
+  text = read_disk(x)
+  xml = Nokogiri.parse(text)
+  return xml
+end
+
+def parse_plain(x)
+  text = read_disk(x)
+  return text
+end
+
+def parse_pdf(x)
+  return Textminer.extract(x)
+end
+
+def is_elsevier_wiley(x)
+  tmp = x.match 'elsevier|wiley'
+  !tmp.nil?
+end

data/lib/textminer/mined.rb

@@ -0,0 +1,31 @@
+require "nokogiri"
+
+##
+# Textminer::Mined
+#
+# Class to give back text mining object
+module Textminer
+  class Mined #:nodoc:
+    attr_accessor :url
+    attr_accessor :path
+    attr_accessor :type
+
+    def initialize(url, path, type)
+      self.url = url
+      self.path = path
+      self.type = type
+    end
+
+    def parse
+      case self.type
+      when 'xml'
+        parse_xml(self.path)
+      when 'plain'
+        parse_plain(self.path)
+      when 'pdf'
+        parse_pdf(self.path)
+      end
+    end
+
+  end
+end

data/lib/textminer/miner.rb

@@ -0,0 +1,42 @@
+require "faraday"
+require "faraday_middleware"
+require "multi_json"
+require 'textminer/helpers/configuration'
+require 'textminer/mined'
+require 'textminer/mine_utils'
+
+##
+# Textminer::Miner
+#
+# Class to give back text mining object
+module Textminer
+  class Miner #:nodoc:
+    attr_accessor :url
+
+    def initialize(url)
+      self.url = url
+    end
+
+    def perform
+      conn = Faraday.new self.url do |c|
+        c.use FaradayMiddleware::FollowRedirects
+        c.adapter :net_http
+      end
+
+      if is_elsevier_wiley(self.url)
+        res = conn.get do |req|
+          req.headers['CR-Clickthrough-Client-Token'] = Textminer.tdm_key
+        end
+      else
+        res = conn.get
+      end
+
+      type = detect_type(res)
+      path = make_path(type)
+      write_disk(res, path)
+
+      return Mined.new(self.url, path, type)
+    end
+
+  end
+end

data/lib/textminer/request.rb

@@ -1,19 +1,36 @@
 module Textminer
   class Request #:nodoc:
     attr_accessor :doi
+    attr_accessor :member
+    attr_accessor :filter
+    attr_accessor :limit
+    attr_accessor :options
 
-    def initialize(doi)
+    def initialize(doi, member, filter, limit, options)
       self.doi = doi
+      self.member = member
+      self.filter = filter
+      self.limit = limit
+      self.options = options
     end
 
     def perform
-      url = "http://api.crossref.org/works/"
-      coll = []
-      Array(self.doi).each do |x|
-        coll << HTTParty.get(url + x)
+      fac = nil
+
+      if member.nil?
+        res = Serrano.works(ids: doi, filter: filter, limit: limit, options: options)
+        if doi.nil?
+          fac = Serrano.works(ids: doi, filter: filter, options: options, facet: 'license:*', limit: 0)
+          fac = fac['message']['facets']['license']['value-count'].to_s
+        end
+      else
+        res = Serrano.members(ids: member, filter: filter, works: true, limit: limit, options: options)
+        if member.nil?
+          fac = Serrano.member(ids: member, filter: filter, options: options, facet: 'license:*', limit: 0)
+          fac = fac['message']['facets']['license']['value-count'].to_s
+        end
       end
-      # res = HTTParty.get(url + self.doi)
-      Response.new(self.doi, coll)
+      Response.new(self.doi, self.member, res, fac)
     end
   end
 end

data/lib/textminer/response.rb

@@ -1,52 +1,76 @@
+require 'launchy'
+require "textminer/link_methods_hash"
+require "textminer/link_methods_array"
+
 module Textminer
   class Response #:nodoc:
-    attr_reader :doi, :response
+    attr_reader :doi, :member, :response, :facet
 
-    def initialize(doi, res)
+    def initialize(doi, member, response, facet)
       @doi = doi
-      @res = res
+      @member = member
+      @response = response
+      @facet = facet
     end
 
-    def raw_body
-      # @res
-      @res.collect { |x| x.body }
+    def to_s
+      if !@doi.nil?
+        if @doi.length > 3
+          ending = '...'
+        else
+          ending = ''
+        end
+        tt = sprintf('dois: %s %s', Array(@doi)[0..2].join(', '), ending)
+      end
+      if !@member.nil?
+        tt = 'member: ' + @member.to_s
+      end
+      if @doi.nil? && @member.nil?
+        tt = ''
+      end
+      sprintf("<textminer>: \n      search: %s\n      no. licenses: %s", tt, @facet)
     end
 
-    def parsed
-      # JSON.parse(@res.body)
-      @res.collect { |x| JSON.parse(x.body) }
+    def inspect
+      to_s
     end
 
-    def links
-      # @res['message']['link']
-      @res.collect { |x| x['message']['link'] }
+    def body
+      @response
     end
 
-    def pdf
-      tmp = links
-      if !tmp.nil?
-        tmp.collect { |z|
-          z.select{ |x| x['content-type'] == "application/pdf" }[0]['URL']
-        }
-      end
+    def links(just_urls = true)
+      tmp = @response.links(just_urls)
+      compactif(tmp)
     end
 
-    def xml
-      tmp = links
-      if !tmp.nil?
-        tmp.collect { |z|
-          z.select{ |x| x['content-type'] == "application/xml" }[0]['URL']
-        }
-      end
+    def links_xml(just_urls = true)
+      tmp = @response.links_xml(just_urls)
+      compactif(tmp)
     end
 
-    def all
-      [xml, pdf]
+    def links_pdf(just_urls = true)
+      tmp = @response.links_pdf(just_urls)
+      compactif(tmp)
     end
 
-    # def browse
+    def links_plain(just_urls = true)
+      tmp = @response.links_plain(just_urls)
+      compactif(tmp)
+    end
 
-    # end
+    protected
 
+    def compactif(z)
+      if z.nil?
+        return z
+      else
+        return z.compact
+      end
+    end
+    # def browse
+    #   url = 'http://doi.org/' + @doi
+    #   Launchy.open(url)
+    # end
   end
 end

data/lib/textminer/tmutils.rb

@@ -0,0 +1,7 @@
+def singlearray2hash(x)
+  if x.length == 1 && x.class == Array
+    return x[0]
+  else
+    return x
+  end
+end

data/lib/textminer/version.rb

@@ -1,3 +1,3 @@
 module Textminer
-  VERSION = "0.1.0"
+  VERSION = "0.1.5"
 end

data/textminer.gemspec

@@ -6,7 +6,7 @@ require 'textminer/version'
 Gem::Specification.new do |s|
   s.name        = 'textminer'
   s.version     = Textminer::VERSION
-  s.date        = '2015-08-24'
+  s.date        = '2015-12-04'
   s.summary     = "Interact with Crossref's Text and Data mining API"
   s.description = "Search Crossref's search API for full text content, and get full text content."
   s.authors     = "Scott Chamberlain"
@@ -15,7 +15,6 @@ Gem::Specification.new do |s|
   s.licenses    = 'MIT'
 
   s.files = `git ls-files -z`.split("\x0").reject {|f| f.match(%r{^(test|spec|features)/}) }
-  s.test_files  = ["test/test_tdm.rb"]
   s.require_paths = ["lib"]
 
   s.bindir      = 'bin'
@@ -27,9 +26,16 @@ Gem::Specification.new do |s|
   s.add_development_dependency "oga", '~> 1.2'
   s.add_development_dependency "simplecov", '~> 0.10'
   s.add_development_dependency "codecov", '~> 0.1'
+
+  s.add_runtime_dependency 'serrano', '~> 0.1.4.1'
   s.add_runtime_dependency 'httparty', '~> 0.13'
   s.add_runtime_dependency 'thor', '~> 0.19'
   s.add_runtime_dependency 'json', '~> 1.8'
-  s.add_runtime_dependency 'launchy', '~> 2.4', '>= 2.4.2'
+  s.add_runtime_dependency 'multi_json', '~> 1.0'
+  s.add_runtime_dependency 'faraday', '~> 0.9.1'
+  s.add_runtime_dependency 'faraday_middleware', '~> 0.10.0'
+  s.add_runtime_dependency 'launchy', '~> 2.4', '>= 2.4.3'
   s.add_runtime_dependency 'pdf-reader','~> 1.3'
+  s.add_runtime_dependency 'nokogiri', '~> 1.6', '>= 1.6.6.2'
+  s.add_runtime_dependency 'uuidtools', '~> 2.1', '>= 2.1.5'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: textminer
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.5
 platform: ruby
 authors:
 - Scott Chamberlain
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-24 00:00:00.000000000 Z
+date: 2015-12-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -94,6 +94,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: serrano
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.4.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.4.1
 - !ruby/object:Gem::Dependency
   name: httparty
   requirement: !ruby/object:Gem::Requirement
@@ -136,6 +150,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.1
+- !ruby/object:Gem::Dependency
+  name: faraday_middleware
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.0
 - !ruby/object:Gem::Dependency
   name: launchy
   requirement: !ruby/object:Gem::Requirement
@@ -145,7 +201,7 @@ dependencies:
         version: '2.4'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.2
+        version: 2.4.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -155,7 +211,7 @@ dependencies:
         version: '2.4'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.4.2
+        version: 2.4.3
 - !ruby/object:Gem::Dependency
   name: pdf-reader
   requirement: !ruby/object:Gem::Requirement
@@ -170,6 +226,46 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.6.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.6.2
+- !ruby/object:Gem::Dependency
+  name: uuidtools
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.5
 description: Search Crossref's search API for full text content, and get full text
   content.
 email: myrmecocystus@gmail.com
@@ -180,18 +276,25 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - Gemfile.lock
-- NEWS.md
 - README.md
 - Rakefile
 - bin/tm
+- extra/fetch.rb
+- extra/fetch_method.rb
 - lib/textminer.rb
-- lib/textminer/fetch.rb
+- lib/textminer/helpers/configuration.rb
+- lib/textminer/link_methods_array.rb
+- lib/textminer/link_methods_hash.rb
+- lib/textminer/mine_utils.rb
+- lib/textminer/mined.rb
+- lib/textminer/miner.rb
 - lib/textminer/request.rb
 - lib/textminer/response.rb
+- lib/textminer/tmutils.rb
 - lib/textminer/version.rb
-- test/test_tdm.rb
 - textminer.gemspec
 homepage: http://github.com/sckott/textminer
 licenses:
@@ -213,10 +316,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: Interact with Crossref's Text and Data mining API
-test_files:
-- test/test_tdm.rb
+test_files: []
 has_rdoc: 

data/NEWS.md

@@ -1,3 +0,0 @@
-## 0.0.1 (2015-08-22)
-
-* First version

data/test/test_tdm.rb

@@ -1,52 +0,0 @@
-require 'simplecov'
-SimpleCov.start
-if ENV['CI']=='true'
-  require 'codecov'
-  SimpleCov.formatter = SimpleCov::Formatter::Codecov
-end
-
-require "textminer"
-require 'fileutils'
-require "test/unit"
-require "oga"
-
-class TestResponse < Test::Unit::TestCase
-
-  def setup
-    @doi = '10.5555/515151'
-    @doi2 = "10.3897/phytokeys.42.7604"
-    @pdf = ["http://annalsofpsychoceramics.labs.crossref.org/fulltext/10.5555/515151.pdf"]
-    @xml = ["http://annalsofpsychoceramics.labs.crossref.org/fulltext/10.5555/515151.xml"]
-  end
-
-  def test_links_endpoint
-    assert_equal(Textminer::Response, Textminer.links(@doi).class)
-  end
-
-  def test_doi
-    assert_equal(@doi, Textminer.links(@doi).doi)
-  end
-
-  def test_pdf
-    assert_equal(@pdf, Textminer.links(@doi).pdf)
-  end
-
-  def test_xml
-    assert_equal(@xml, Textminer.links(@doi).xml)
-  end
-
-  def test_fetch_xml
-    res = Textminer.fetch(@doi2, "xml")
-    assert_equal(HTTParty::Response, res[0].class)
-    assert_true(res[0].ok?)
-    assert_equal(String, res[0].body.class)
-    assert_equal("PhytoKeys", Oga.parse_xml(res[0].body).xpath('//journal-meta//journal-id').text)
-  end
-
-  # def test_fetch_pdf
-  #   res = Textminer.fetch(@doi2, "pdf")
-  #   assert_equal(HTTParty::Response, res.class)
-  #   assert_true(res.ok?)
-  # end
-
-end