textminer 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5e954930af35bca6c9752b9f9d660eb675cd4bea
+  data.tar.gz: c451255b116b2eae5a52d66adc3619e39c7f10c4
+SHA512:
+  metadata.gz: 0ff5aacaf4be3b3a797f6cc8435c9c4a2bec7be98d695986f2b27a044b98492f5729cd58979952dacf96777d2306c2c8b9d9abda8dc7bfa94e9080a4f4ae8f6c
+  data.tar.gz: 64a5fd5ebb268403c12350d444794efde329e97fb14e7ad47959d4af5aa3465306f135174e183b4ce11ab1009ba24fc29ebc47d7b605e90a4e4e308274264671

data/.gitignore

@@ -0,0 +1,36 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+#Gemfile.lock
+.ruby-version
+.ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+cache/

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.1.7
+  - 2.2.3

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,68 @@
+PATH
+  remote: .
+  specs:
+    textminer (0.1.0)
+      httparty (~> 0.13)
+      json (~> 1.8)
+      launchy (~> 2.4, >= 2.4.2)
+      pdf-reader (~> 1.3)
+      thor (~> 0.19)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    Ascii85 (1.0.2)
+    addressable (2.3.8)
+    afm (0.2.2)
+    ansi (1.5.0)
+    ast (2.1.0)
+    codecov (0.1.1)
+      json
+      simplecov
+      url
+    docile (1.1.5)
+    hashery (2.1.1)
+    httparty (0.13.5)
+      json (~> 1.8)
+      multi_xml (>= 0.5.2)
+    json (1.8.3)
+    launchy (2.4.3)
+      addressable (~> 2.3)
+    multi_xml (0.5.5)
+    oga (1.2.3)
+      ast
+      ruby-ll (~> 2.1)
+    pdf-reader (1.3.3)
+      Ascii85 (~> 1.0.0)
+      afm (~> 0.2.0)
+      hashery (~> 2.0)
+      ruby-rc4
+      ttfunk
+    power_assert (0.2.4)
+    rake (10.4.2)
+    ruby-ll (2.1.2)
+      ansi
+      ast
+    ruby-rc4 (0.1.5)
+    simplecov (0.10.0)
+      docile (~> 1.1.0)
+      json (~> 1.8)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
+    test-unit (3.1.3)
+      power_assert
+    thor (0.19.1)
+    ttfunk (1.4.0)
+    url (0.3.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.6)
+  codecov (~> 0.1)
+  oga (~> 1.2)
+  rake (~> 10.4)
+  simplecov (~> 0.10)
+  test-unit (~> 3.1)
+  textminer!

data/NEWS.md

@@ -0,0 +1,3 @@
+## 0.0.1 (2015-08-22)
+
+* First version

data/README.md

@@ -0,0 +1,116 @@
+textminer
+=========
+
+[![Build Status](https://api.travis-ci.org/sckott/textminer.png)](https://travis-ci.org/sckott/textminer)
+[![codecov.io](http://codecov.io/github/sckott/textminer/coverage.svg?branch=master)](http://codecov.io/github/sckott/textminer?branch=master)
+
+__This is alpha software, so expect changes__
+
+## What is it?
+
+__`textminer` helps you text mine through Crossref's TDM (Text & Data Mining) services:__
+
+## Changes
+
+For changes see the [NEWS file](https://github.com/sckott/textminer/blob/master/NEWS.md).
+
+## Install
+
+### Release version
+
+Not on RubyGems yet
+
+### Development version
+
+```
+git clone git@github.com:sckott/textminer.git
+cd textminer
+rake install
+```
+
+## Within Ruby
+
+Search by DOI
+
+```ruby
+require 'textminer'
+out = textminer.links("10.5555/515151")
+```
+
+Get the pdf link
+
+```ruby
+out.pdf
+```
+
+```ruby
+"http://annalsofpsychoceramics.labs.crossref.org/fulltext/10.5555/515151.pdf"
+```
+
+Get the xml link
+
+```ruby
+out.xml
+```
+
+```ruby
+"http://annalsofpsychoceramics.labs.crossref.org/fulltext/10.5555/515151.xml"
+```
+
+Fetch XML
+
+```ruby
+Textminer.fetch("10.3897/phytokeys.42.7604", "xml")
+```
+
+```ruby
+=> {"article"=>
+  {"front"=>
+    {"journal_meta"=>
+      {"journal_id"=>
+        {"__content__"=>"PhytoKeys", "journal_id_type"=>"publisher-id"},
+       "journal_title_group"=>
+        {"journal_title"=>{"__content__"=>"PhytoKeys", "lang"=>"en"},
+         "abbrev_journal_title"=>{"__content__"=>"PhytoKeys", "lang"=>"en"}},
+       "issn"=>
+        [{"__content__"=>"1314-2011", "pub_type"=>"ppub"},
+         {"__content__"=>"1314-2003", "pub_type"=>"epub"}],
+       "publisher"=>{"publisher_name"=>"Pensoft Publishers"}},
+     "article_meta"=>
+
+...
+```
+
+Fetch PDF
+
+```ruby
+Textminer.fetch("10.3897/phytokeys.42.7604", "pdf")
+```
+
+> pdf written to disk
+
+## On the CLI
+
+Get links
+
+```sh
+tm links 10.3897/phytokeys.42.7604
+```
+
+```sh
+http://phytokeys.pensoft.net/lib/ajax_srv/article_elements_srv.php?action=download_xml&item_id=4190
+http://phytokeys.pensoft.net/lib/ajax_srv/article_elements_srv.php?action=download_pdf&item_id=4190
+```
+
+More than one DOI:
+
+```sh
+tm links '10.3897/phytokeys.42.7604,10.3897/zookeys.516.9439'
+```
+
+## To do
+
+* CLI executable
+* get actual full text
+* better test suite
+* documentation

data/Rakefile

@@ -0,0 +1,26 @@
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList['test/test*.rb']
+  t.verbose = true
+end
+
+desc "Run tests"
+task :default => :test
+
+desc "Build textminer"
+task :build do
+	system "gem build textminer.gemspec"
+end
+
+desc "Install textminer"
+task :install => :build do
+	system "gem install textminer-#{Textminer::VERSION}.gem"
+end
+
+desc "Release to Rubygems"
+task :release => :build do
+  system "gem push textminer-#{Textminer::VERSION}.gem"
+end

data/bin/tm

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+
+require "thor"
+
+class Tm < Thor
+  include Thor::Actions
+  require 'textminer'
+  require 'launchy'
+  require 'json'
+  require 'pp'
+
+  desc "links STRING", "Get links meant for text mining"
+  # method_option :doi => :string
+  def links(tt)
+  	tt = "#{tt}"
+    tt = tt.to_s.split(',')
+    out = Textminer.links(tt).all
+    puts out
+  end
+end
+
+Tm.start(ARGV)

data/lib/textminer.rb

@@ -0,0 +1,66 @@
+require 'httparty'
+require 'json'
+require 'pdf-reader'
+require "textminer/version"
+require "textminer/request"
+require "textminer/response"
+require "textminer/fetch"
+
+module Textminer
+  ##
+  # Get links meant for text mining
+  #
+  # @param doi [Array] A DOI, digital object identifier
+  # @return [Array] the output
+  #
+  # @example
+  #     require 'textminer'
+  #     # link to full text available
+  #     Textminer.links("10.5555/515151")
+  #     # no link to full text available
+  #     Textminer.links("10.1371/journal.pone.0000308")
+  #     # many DOIs at once
+  #     res = Textminer.links(["10.3897/phytokeys.42.7604", "10.3897/zookeys.516.9439"])
+  #     res.links
+  #     res.pdf
+  #     res.xml
+  def self.links(doi)
+    Request.new(doi).perform
+  end
+
+  ##
+  # Thin layer around pdf-reader gem's PDF::Reader
+  #
+  # @param doi [Array] A DOI, digital object identifier
+  # @param type [Array] One of two options to download: xml (default) or pdf
+  #
+  # @example
+  #     require 'textminer'
+  #     # fetch full text by DOI - xml by default
+  #     Textminer.fetch("10.3897/phytokeys.42.7604")
+  #     # many DOIs - xml output
+  #     res = Textminer.fetch(["10.3897/phytokeys.42.7604", "10.3897/zookeys.516.9439"])
+  #     # fetch full text - pdf
+  #     Textminer.fetch("10.3897/phytokeys.42.7604", "pdf")
+  def self.fetch(doi, type = 'xml')
+    Fetch.new(doi, type).fetchtext
+  end
+
+  ##
+  # Thin layer around pdf-reader gem's PDF::Reader
+  #
+  # @param path [String] Path to a pdf file downloaded via {fetch}, or
+  #   another way.
+  #
+  # @example
+  #     require 'textminer'
+  #     # fetch full text - pdf
+  #     res = Textminer.fetch("10.3897/phytokeys.42.7604", "pdf")
+  #     # extract pdf to text
+  #     Textminer.extract(res)
+  def self.extract(path)
+    rr = PDF::Reader.new(path)
+    rr.pages.map { |page| page.text }.join("\n")
+  end
+
+end

data/lib/textminer/fetch.rb

@@ -0,0 +1,49 @@
+module Textminer
+  class Fetch #:nodoc:
+      attr_accessor :doi, :type
+
+      def initialize(doi, type)
+        self.doi = doi
+        self.type = type
+      end
+
+      def fetchtext
+        lks = Textminer.links(self.doi)
+        lk = pick_link(lks)
+        case self.type
+        when "xml"
+          # HTTParty.get(lk)
+          coll = []
+          Array(lk).each do |x|
+            coll << HTTParty.get(x)
+          end
+          return coll
+        when "pdf"
+          serialize_pdf(lk, self.doi)
+        end
+      end
+
+      private
+
+      def pick_link(x)
+        case self.type
+        when "xml"
+          x.xml
+        when "pdf"
+          x.pdf
+        else
+          puts "type must be xml or pdf"
+        end
+      end
+
+      def serialize_pdf(x, y)
+        path = "/Users/sacmac/.textminer/" + y.gsub('/', '_') + ".pdf"
+        File.open(path, "wb") do |f|
+          f.write HTTParty.get(x).parsed_response
+        end
+
+        return path
+      end
+
+    end
+end

data/lib/textminer/request.rb

@@ -0,0 +1,19 @@
+module Textminer
+  class Request #:nodoc:
+    attr_accessor :doi
+
+    def initialize(doi)
+      self.doi = doi
+    end
+
+    def perform
+      url = "http://api.crossref.org/works/"
+      coll = []
+      Array(self.doi).each do |x|
+        coll << HTTParty.get(url + x)
+      end
+      # res = HTTParty.get(url + self.doi)
+      Response.new(self.doi, coll)
+    end
+  end
+end

data/lib/textminer/response.rb

@@ -0,0 +1,52 @@
+module Textminer
+  class Response #:nodoc:
+    attr_reader :doi, :response
+
+    def initialize(doi, res)
+      @doi = doi
+      @res = res
+    end
+
+    def raw_body
+      # @res
+      @res.collect { |x| x.body }
+    end
+
+    def parsed
+      # JSON.parse(@res.body)
+      @res.collect { |x| JSON.parse(x.body) }
+    end
+
+    def links
+      # @res['message']['link']
+      @res.collect { |x| x['message']['link'] }
+    end
+
+    def pdf
+      tmp = links
+      if !tmp.nil?
+        tmp.collect { |z|
+          z.select{ |x| x['content-type'] == "application/pdf" }[0]['URL']
+        }
+      end
+    end
+
+    def xml
+      tmp = links
+      if !tmp.nil?
+        tmp.collect { |z|
+          z.select{ |x| x['content-type'] == "application/xml" }[0]['URL']
+        }
+      end
+    end
+
+    def all
+      [xml, pdf]
+    end
+
+    # def browse
+
+    # end
+
+  end
+end

data/lib/textminer/version.rb

@@ -0,0 +1,3 @@
+module Textminer
+  VERSION = "0.1.0"
+end

data/test/test_tdm.rb

@@ -0,0 +1,52 @@
+require 'simplecov'
+SimpleCov.start
+if ENV['CI']=='true'
+  require 'codecov'
+  SimpleCov.formatter = SimpleCov::Formatter::Codecov
+end
+
+require "textminer"
+require 'fileutils'
+require "test/unit"
+require "oga"
+
+class TestResponse < Test::Unit::TestCase
+
+  def setup
+    @doi = '10.5555/515151'
+    @doi2 = "10.3897/phytokeys.42.7604"
+    @pdf = ["http://annalsofpsychoceramics.labs.crossref.org/fulltext/10.5555/515151.pdf"]
+    @xml = ["http://annalsofpsychoceramics.labs.crossref.org/fulltext/10.5555/515151.xml"]
+  end
+
+  def test_links_endpoint
+    assert_equal(Textminer::Response, Textminer.links(@doi).class)
+  end
+
+  def test_doi
+    assert_equal(@doi, Textminer.links(@doi).doi)
+  end
+
+  def test_pdf
+    assert_equal(@pdf, Textminer.links(@doi).pdf)
+  end
+
+  def test_xml
+    assert_equal(@xml, Textminer.links(@doi).xml)
+  end
+
+  def test_fetch_xml
+    res = Textminer.fetch(@doi2, "xml")
+    assert_equal(HTTParty::Response, res[0].class)
+    assert_true(res[0].ok?)
+    assert_equal(String, res[0].body.class)
+    assert_equal("PhytoKeys", Oga.parse_xml(res[0].body).xpath('//journal-meta//journal-id').text)
+  end
+
+  # def test_fetch_pdf
+  #   res = Textminer.fetch(@doi2, "pdf")
+  #   assert_equal(HTTParty::Response, res.class)
+  #   assert_true(res.ok?)
+  # end
+
+end

data/textminer.gemspec

@@ -0,0 +1,35 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'textminer/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'textminer'
+  s.version     = Textminer::VERSION
+  s.date        = '2015-08-24'
+  s.summary     = "Interact with Crossref's Text and Data mining API"
+  s.description = "Search Crossref's search API for full text content, and get full text content."
+  s.authors     = "Scott Chamberlain"
+  s.email       = 'myrmecocystus@gmail.com'
+  s.homepage    = 'http://github.com/sckott/textminer'
+  s.licenses    = 'MIT'
+
+  s.files = `git ls-files -z`.split("\x0").reject {|f| f.match(%r{^(test|spec|features)/}) }
+  s.test_files  = ["test/test_tdm.rb"]
+  s.require_paths = ["lib"]
+
+  s.bindir      = 'bin'
+  s.executables = ['tm']
+
+  s.add_development_dependency "bundler", '~> 1.6'
+  s.add_development_dependency "rake", '~> 10.4'
+  s.add_development_dependency "test-unit", '~> 3.1'
+  s.add_development_dependency "oga", '~> 1.2'
+  s.add_development_dependency "simplecov", '~> 0.10'
+  s.add_development_dependency "codecov", '~> 0.1'
+  s.add_runtime_dependency 'httparty', '~> 0.13'
+  s.add_runtime_dependency 'thor', '~> 0.19'
+  s.add_runtime_dependency 'json', '~> 1.8'
+  s.add_runtime_dependency 'launchy', '~> 2.4', '>= 2.4.2'
+  s.add_runtime_dependency 'pdf-reader','~> 1.3'
+end

metadata

@@ -0,0 +1,222 @@
+--- !ruby/object:Gem::Specification
+name: textminer
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Scott Chamberlain
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-08-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.4'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: oga
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: codecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.13'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.13'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.19'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+- !ruby/object:Gem::Dependency
+  name: launchy
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.4.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.4.2
+- !ruby/object:Gem::Dependency
+  name: pdf-reader
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+description: Search Crossref's search API for full text content, and get full text
+  content.
+email: myrmecocystus@gmail.com
+executables:
+- tm
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- NEWS.md
+- README.md
+- Rakefile
+- bin/tm
+- lib/textminer.rb
+- lib/textminer/fetch.rb
+- lib/textminer/request.rb
+- lib/textminer/response.rb
+- lib/textminer/version.rb
+- test/test_tdm.rb
+- textminer.gemspec
+homepage: http://github.com/sckott/textminer
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Interact with Crossref's Text and Data mining API
+test_files:
+- test/test_tdm.rb
+has_rdoc: