yahoo_content_analysis 0.0.1

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.key
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in yahoo_content_analysis.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Andrew Kuklewicz
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,42 @@
+# YahooContentAnalysis
+
+Use the Yahoo! Content Analysis API to extract topics and entities.
+
+OAuth access is implemented, though not necessary (the docs imply higher limitd if requests are signed).  This should be a useful example for those wanting to do 2-legged OAuth access to Yahoo APIs using Faraday and the OAuth Faraday Middleware.
+
+N.B. - I am not seeing the additional metadata nor related entities returned as the Yahoo docs claim they should.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'yahoo_content_analysis'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install yahoo_content_analysis
+
+## Usage
+```ruby
+require 'yahoo_content_analysis'
+
+YahooContentAnalysis.configure{|y| 
+  y.api_key = ENV['YAHOO_API_KEY']
+  y.api_secret = ENV['YAHOO_API_SECRET']
+}
+
+r = YahooContentAnalysis::Client.new.analyze('Italian sculptors and painters of the renaissance favored the Virgin Mary for inspiration.')
+
+names = r.entities.collect{|e| e.name}
+```
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/yahoo_content_analysis/client.rb

@@ -0,0 +1,69 @@
+# encoding: utf-8
+
+module YahooContentAnalysis
+  class Client
+
+    include Connection
+
+    attr_reader *YahooContentAnalysis::Configuration.keys
+
+    attr_accessor :current_options
+
+    class_eval do
+      YahooContentAnalysis::Configuration.keys.each do |key|
+        define_method "#{key}=" do |arg|
+          self.instance_variable_set("@#{key}", arg)
+          self.current_options.merge!({:"#{key}" => arg})
+        end
+      end
+    end
+
+    def initialize(options={}, &block)
+      setup(options)
+      yield(self) if block_given?
+    end
+
+    def setup(options={})
+      options = YahooContentAnalysis.options.merge(options)
+      self.current_options = options
+      Configuration.keys.each do |key|
+        send("#{key}=", options[key])
+      end
+    end
+
+    def analyze(content, opts={})      
+      raise 'Specify a value for the content' unless content
+      response = connection.post do |request|
+        request.params = options(content)
+      end
+      YahooContentAnalysis::Response.new(response)
+    end
+
+    def query(content)
+      "SELECT * FROM contentanalysis.analyze WHERE related_entities = \"true\" and #{condition(content)}"
+    end
+
+    def condition(content)
+      content.is_a?(URI) ? url(content) : text(content)
+    end
+
+    def url(content)
+      %{ url = "#{content}" }
+    end
+
+    def text(content)
+      %{ text = "#{content.gsub('"', '\"')}" }
+    end
+
+    def options(content)
+      {
+        'q'                => query(content),
+        'format'           => 'json',
+        'max'              => '50',
+        'related_entities' => 'true',
+        'show_metadata'    => 'true'
+      }
+    end
+
+  end
+end

data/lib/yahoo_content_analysis/configuration.rb

@@ -0,0 +1,103 @@
+# encoding: utf-8
+
+# these are additional params from the docs, figure out how to use them - some don't seem to do squat.
+# related_entities    boolean: true (default), false  Whether or not to include related entities/concepts in the response
+# show_metadata       boolean: true (default), false  Whether or not to include entity/concept metadata in the response
+# enable_categorizer  boolean: true (default), false  Whether or not to include document category information in the response
+# unique              boolean: true, false (default)  Whether or not to detect only one occurrence of an entity or a concept that my appear multiple times
+# max                 integer: 100 (default)          Maximum number of entities/concepts to detect
+
+module YahooContentAnalysis
+
+  module Configuration
+
+    VALID_OPTIONS_KEYS = [
+      :api_key,
+      :api_secret,
+      :adapter,
+      :endpoint,
+      :user_agent,
+      :format,
+      :max,
+      :related_entities,
+      :show_metadata,
+      :enable_categorizer,
+      :unique
+    ].freeze
+
+    VALID_PARAMS = [
+      :format,
+      :max,
+      :related_entities,
+      :show_metadata,
+      :enable_categorizer,
+      :unique
+    ]
+
+    # this you need to get from yahoo - go register an app!
+    DEFAULT_API_KEY = nil
+
+    DEFAULT_API_SECRET = nil
+
+    # Adapters are whatever Faraday supports - I like excon alot, so I'm defaulting it
+    DEFAULT_ADAPTER = :excon
+
+    # The api endpoint for YQL
+    DEFAULT_ENDPOINT = 'http://query.yahooapis.com/v1/public/yql'.freeze
+
+    # The value sent in the http header for 'User-Agent' if none is set
+    DEFAULT_USER_AGENT = "YahooContentAnalysis Ruby Gem #{YahooContentAnalysis::VERSION}".freeze
+
+    DEFAULT_FORMAT = :json
+
+    DEFAULT_MAX = 50
+
+    DEFAULT_RELATED_ENTITIES = 'true'
+
+    DEFAULT_SHOW_METADATA = 'true'
+
+    DEFAULT_ENABLE_CATEGORIZER = 'true'
+
+    DEFAULT_UNIQUE = 'true'
+
+    attr_accessor *VALID_OPTIONS_KEYS
+
+    # Convenience method to allow for global setting of configuration options
+    def configure
+      yield self
+    end
+
+    def self.extended(base)
+      base.reset!
+    end
+
+    class << self
+      def keys
+        VALID_OPTIONS_KEYS
+      end
+    end
+
+    def options
+      options = {}
+      VALID_OPTIONS_KEYS.each { |k| options[k] = send(k) }
+      options
+    end
+
+    # Reset configuration options to their defaults
+    def reset!
+      self.api_key            = DEFAULT_API_KEY
+      self.api_secret         = DEFAULT_API_SECRET
+      self.adapter            = DEFAULT_ADAPTER
+      self.endpoint           = DEFAULT_ENDPOINT
+      self.user_agent         = DEFAULT_USER_AGENT
+      self.format             = DEFAULT_FORMAT
+      self.max                = DEFAULT_MAX
+      self.related_entities   = DEFAULT_RELATED_ENTITIES
+      self.show_metadata      = DEFAULT_SHOW_METADATA
+      self.enable_categorizer = DEFAULT_ENABLE_CATEGORIZER
+      self.unique             = DEFAULT_UNIQUE
+      self
+    end
+
+  end
+end

data/lib/yahoo_content_analysis/connection.rb

@@ -0,0 +1,55 @@
+# encoding: utf-8
+
+require 'faraday_middleware'
+
+module YahooContentAnalysis
+  module Connection
+
+    ALLOWED_OPTIONS = [
+      :headers,
+      :url,
+      :params,
+      :request,
+      :ssl
+    ].freeze
+
+    def add_default_options(opts={})
+      headers = opts.delete(:headers) || {}
+      options = {
+        :consumer_key => api_key,
+        :consumer_secret => api_secret,
+        :headers => {
+          # generic http headers
+          'User-Agent'   => user_agent,
+          'Accept'       => "application/json;charset=utf-8"
+        },
+        :ssl => {:verify => false},
+        :url => endpoint
+      }.merge(opts)
+      options[:headers] = options[:headers].merge(headers)
+      options
+    end
+
+    def connection(options={})
+      opts = add_default_options(options)
+      oauth = oauth_opts(opts)
+      Faraday::Connection.new(opts) do |connection|
+        connection.request :oauth, oauth if oauth
+        connection.request :url_encoded
+        connection.response :mashify
+        connection.response :logger if ENV['DEBUG']
+        connection.response :json
+        connection.adapter(adapter)
+      end
+    end
+
+    def oauth_opts(opts)
+      oauth_opts =  if opts.key?(:consumer_secret) && opts.key?(:consumer_key)
+        {:consumer_secret => opts.delete(:consumer_secret), :consumer_key => opts.delete(:consumer_key)}
+      else
+        nil
+      end
+    end
+
+  end
+end

data/lib/yahoo_content_analysis/response.rb

@@ -0,0 +1,58 @@
+# encoding: utf-8
+require 'language_list'
+
+module YahooContentAnalysis
+  class Response
+    attr_accessor :raw, :language, :topics, :tags, :entities, :relations, :locations
+
+    def initialize(response)
+      @raw  = response
+
+      @language = nil
+      @topics    = []
+      @tags      = []
+      @entities  = []
+      @relations = []
+      @locations = []
+
+      parse(response)
+    end
+
+    def parse(response)
+      r = response.body['query']['results'] || {} 
+      @language = get_language(r['lang'])
+
+      @topics = Array((r['yctCategories'] || {})['yctCategory']).collect do |cat|
+        {:name => cat['content'], :score => cat['score']}
+      end
+
+      @entities = Array((r['entities'] || {})['entity']).collect do |ent|
+        type = extract_type(ent['types'])
+
+        entity = {:name => ent['text']['content'], :score => ent['score']}
+        entity[:type] = type if type
+        entity[:wikipedia_url] = ent['wiki_url'] if ent['wiki_url']
+
+        ## these aren't showing up in any results, so not worrying about them
+        # if cat['related_entities'] && cat['related_entities']['wikipedia']
+        # end
+
+        entity
+      end
+
+    end
+
+    def get_language(lang)
+      return nil unless lang
+      l = LanguageList::LanguageInfo.find(lang.split('-')[0].downcase)
+      l.name
+    end
+
+    def extract_type(h)
+      return nil unless (h && h['type'])
+      type = h['type'].is_a?(Array) ? h['type'].first : h['type']
+      (type['content'] || '').sub(/^\/(.*)/, '')
+    end
+
+  end
+end

data/lib/yahoo_content_analysis/version.rb

@@ -0,0 +1,5 @@
+# encoding: utf-8
+
+module YahooContentAnalysis
+  VERSION = "0.0.1"
+end

data/lib/yahoo_content_analysis.rb

@@ -0,0 +1,12 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'yahoo_content_analysis/version'
+require 'yahoo_content_analysis/configuration'
+require 'yahoo_content_analysis/connection'
+require 'yahoo_content_analysis/response'
+require 'yahoo_content_analysis/client'
+
+module YahooContentAnalysis
+  extend Configuration
+end

data/test/client_test.rb

@@ -0,0 +1,7 @@
+require 'yahoo_content_analysis'
+YahooContentAnalysis.configure{|y| 
+  y.api_key = ENV['YAHOO_API_KEY']
+  y.api_secret = ENV['YAHOO_API_SECRET']
+}
+
+r = YahooContentAnalysis::Client.new.analyze('Italian sculptors and painters of the renaissance favored the Virgin Mary for inspiration.')

data/yahoo_content_analysis.gemspec

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'yahoo_content_analysis/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "yahoo_content_analysis"
+  gem.version       = YahooContentAnalysis::VERSION
+  gem.authors       = ["Andrew Kuklewicz"]
+  gem.email         = ["andrew@prx.org"]
+  gem.description   = %q{Use the Yahoo! Content Analysis API to extract topics and entities.}
+  gem.summary       = %q{Use the Yahoo! Content Analysis API to extract topics and entities.}
+  gem.homepage      = "https://github.com/PRX/yahoo_content_analysis"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.add_runtime_dependency('faraday', ['>= 0.7.4', '< 0.9'])
+  gem.add_runtime_dependency('faraday_middleware', '~> 0.9')
+  gem.add_runtime_dependency('multi_json', '>= 1.0.3', '~> 1.0')
+  gem.add_runtime_dependency('excon')
+  gem.add_runtime_dependency('hashie',  '>= 0.4.0')
+  gem.add_runtime_dependency('simple_oauth')
+  gem.add_runtime_dependency('language_list')
+
+  gem.add_development_dependency('rake')
+  gem.add_development_dependency('minitest')
+end

metadata

@@ -0,0 +1,221 @@
+--- !ruby/object:Gem::Specification
+name: yahoo_content_analysis
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Andrew Kuklewicz
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-04-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.7.4
+    - - <
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.7.4
+    - - <
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: faraday_middleware
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.3
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.3
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: excon
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: hashie
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+- !ruby/object:Gem::Dependency
+  name: simple_oauth
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: language_list
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Use the Yahoo! Content Analysis API to extract topics and entities.
+email:
+- andrew@prx.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/yahoo_content_analysis.rb
+- lib/yahoo_content_analysis/client.rb
+- lib/yahoo_content_analysis/configuration.rb
+- lib/yahoo_content_analysis/connection.rb
+- lib/yahoo_content_analysis/response.rb
+- lib/yahoo_content_analysis/version.rb
+- test/client_test.rb
+- yahoo_content_analysis.gemspec
+homepage: https://github.com/PRX/yahoo_content_analysis
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 2699003970576411257
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 2699003970576411257
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Use the Yahoo! Content Analysis API to extract topics and entities.
+test_files:
+- test/client_test.rb