alchemy_api 0.1.0

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,24 @@
+## MAC OS
+.DS_Store
+
+## TEXTMATE
+*.tmproj
+tmtags
+
+## EMACS
+*~
+\#*
+.\#*
+
+## VIM
+*.swp
+
+## PROJECT::GENERAL
+coverage
+rdoc
+pkg
+.yardoc
+doc
+
+## PROJECT::SPECIFIC
+README.markdown.html

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 David Balatero
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,36 @@
+alchemy_api
+===========
+
+This is a library implementing the AlchemyAPI. It's built on top of [monster_mash](http://github.com/dbalatero/monster_mash), which means you can hit Alchemy with your choice of serial or parallel requests.
+
+Supported
+---------
+* Text extraction from text/html/URL
+* Language detection from text/html/URL
+* Keyword extraction from text/html/URL
+* Text extraction from text/html/URL
+
+Not supported
+-------------
+Currently the following APIs are not supported yet:
+
+* Entity extraction (in progress on branch entities)
+* Content scraping
+* Microformats
+* RSS/ATOM feed detecion
+
+Note on Patches/Pull Requests
+-----------------------------
+
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+Copyright
+---------
+
+Copyright (c) 2010 David Balatero. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,47 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "alchemy_api"
+    gem.summary = %Q{Provides a client API library for AlchemyAPI's awesome NLP services. Allows you to make parallel or serial requests.}
+    gem.description = %Q{Provides a client API library for AlchemyAPI's awesome NLP services. Allows you to make parallel or serial requests.}
+    gem.email = "dbalatero@gmail.com"
+    gem.homepage = "http://github.com/dbalatero/alchemy_api"
+    gem.authors = ["David Balatero"]
+    gem.add_development_dependency "rspec", ">= 1.2.9"
+    gem.add_dependency "monster_mash", ">= 0.1.1"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+  spec.rcov_opts = ['--exclude', 'spec']
+end
+
+task :spec => :check_dependencies
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "alchemy_api #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/alchemy_api.gemspec

@@ -0,0 +1,92 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{alchemy_api}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["David Balatero"]
+  s.date = %q{2010-05-17}
+  s.description = %q{Provides a client API library for AlchemyAPI's awesome NLP services. Allows you to make parallel or serial requests.}
+  s.email = %q{dbalatero@gmail.com}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.markdown",
+     "README.markdown.html"
+  ]
+  s.files = [
+    ".document",
+     ".gitignore",
+     "LICENSE",
+     "README.markdown",
+     "Rakefile",
+     "VERSION",
+     "alchemy_api.gemspec",
+     "lib/alchemy_api.rb",
+     "lib/alchemy_api/base.rb",
+     "lib/alchemy_api/categorization.rb",
+     "lib/alchemy_api/language_detection.rb",
+     "lib/alchemy_api/term_extraction.rb",
+     "lib/alchemy_api/text_extraction.rb",
+     "spec/alchemy_api/base_spec.rb",
+     "spec/alchemy_api/categorization_spec.rb",
+     "spec/alchemy_api/language_detection_spec.rb",
+     "spec/alchemy_api/term_extraction_spec.rb",
+     "spec/alchemy_api/text_extraction_spec.rb",
+     "spec/alchemy_api_spec.rb",
+     "spec/cache/categorization/get_categorization_from_html/ddc3cf50efe5bd5c2159abfb49121cfa2314ca88.cache",
+     "spec/cache/categorization/get_categorization_from_text/8b476a3b532afd2da646b145e9dde07570c27352.cache",
+     "spec/cache/categorization/get_categorization_from_url/7536a34e1d54a95d8ee07d2a98036362761e1621.cache",
+     "spec/cache/language_detection/get_language_from_html/0faf7be978647b611d9c59e1efa497dd76e542f5.cache",
+     "spec/cache/language_detection/get_language_from_text/1ad3f50c1fda37000e24c196f12212ea9d536cb4.cache",
+     "spec/cache/language_detection/get_language_from_url/d077a95e60be0876bb7650ad213f5f43e83454d4.cache",
+     "spec/cache/term_extraction/get_ranked_keywords_from_html/7718a0fbd03739e4213a4e66c32a79a10c3499c3.cache",
+     "spec/cache/term_extraction/get_ranked_keywords_from_text/6f49e68ee4a9150368e671e70b632dbdc40860bb.cache",
+     "spec/cache/term_extraction/get_ranked_keywords_from_url/b9c291523159563d2224d676ec43b7b79a902d21.cache",
+     "spec/cache/text_extraction/get_raw_text_from_html/9db19f848a798db1f9a8c6cce9074d03cf2637a8.cache",
+     "spec/cache/text_extraction/get_raw_text_from_html/e9c236b6e861b57d238c810bb3c307cada170cad.cache",
+     "spec/cache/text_extraction/get_raw_text_from_url/8f5dff27211163e41ea5e7c3c534acf7b87d2098.cache",
+     "spec/cache/text_extraction/get_text_from_html/e7e6dba4c8570a41dbcb05233793018fc5ae4e1e.cache",
+     "spec/cache/text_extraction/get_text_from_url/13facbfeae029d936c7dc18ecaff5d2764b94618.cache",
+     "spec/cache/text_extraction/get_title_from_html/2a526348db23f992fee293d34f94c087e77290c5.cache",
+     "spec/cache/text_extraction/get_title_from_url/e84c0c7c67668706ae0cf3eefcd88c0911cd2b65.cache",
+     "spec/fixtures/article.txt",
+     "spec/fixtures/bp_spill.html",
+     "spec/spec.opts",
+     "spec/spec_helper.rb"
+  ]
+  s.homepage = %q{http://github.com/dbalatero/alchemy_api}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.6}
+  s.summary = %q{Provides a client API library for AlchemyAPI's awesome NLP services. Allows you to make parallel or serial requests.}
+  s.test_files = [
+    "spec/alchemy_api/base_spec.rb",
+     "spec/alchemy_api/categorization_spec.rb",
+     "spec/alchemy_api/language_detection_spec.rb",
+     "spec/alchemy_api/term_extraction_spec.rb",
+     "spec/alchemy_api/text_extraction_spec.rb",
+     "spec/alchemy_api_spec.rb",
+     "spec/spec_helper.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<rspec>, [">= 1.2.9"])
+      s.add_runtime_dependency(%q<monster_mash>, [">= 0.1.1"])
+    else
+      s.add_dependency(%q<rspec>, [">= 1.2.9"])
+      s.add_dependency(%q<monster_mash>, [">= 0.1.1"])
+    end
+  else
+    s.add_dependency(%q<rspec>, [">= 1.2.9"])
+    s.add_dependency(%q<monster_mash>, [">= 0.1.1"])
+  end
+end
+

data/lib/alchemy_api.rb

@@ -0,0 +1,29 @@
+require 'ostruct'
+require 'json'
+require 'monster_mash'
+
+Dir.glob(File.dirname(__FILE__) + "/**/*.rb").each do |f|
+  require f
+end
+
+module AlchemyApi
+  @api_key = nil
+  @base_uri = "http://access.alchemyapi.com/calls/url"
+  @base_html_uri = "http://access.alchemyapi.com/calls/html"
+  @base_text_uri = "http://access.alchemyapi.com/calls/text"
+
+  class << self
+    attr_accessor :api_key
+    attr_accessor :base_uri
+    attr_accessor :base_html_uri
+    attr_accessor :base_text_uri
+  end
+
+  class UnknownError < StandardError; end
+  class InvalidApiKeyError < StandardError; end
+  class CannotRetrieveUrlError < StandardError; end
+  class RedirectionLimitError < CannotRetrieveUrlError; end
+  class PageIsNotValidHtmlError < StandardError; end
+  class ContentExceedsMaxLimitError < StandardError; end
+  class InvalidHtmlError < StandardError; end
+end

data/lib/alchemy_api/base.rb

@@ -0,0 +1,37 @@
+module AlchemyApi
+  class Base < MonsterMash::Base
+    defaults do
+      cache_timeout 999999
+      user_agent 'Ruby AlchemyApi'
+      params :apikey => AlchemyApi.api_key,
+             :outputMode => 'json'
+    end
+
+    def self.get_json(response)
+      json = JSON.parse(response.body)
+      check_json_for_errors_and_raise!(json)
+      json
+    end
+
+    def self.check_json_for_errors_and_raise!(json)
+      if json['status'] == 'ERROR'
+        case json['statusInfo']
+        when 'invalid-api-key'
+          raise InvalidApiKeyError, "The API key you sent (#{AlchemyApi.api_key.inspect}) is invalid! Please set AlchemyApi.api_key!"
+        when 'cannot-retrieve'
+          raise CannotRetrieveUrlError, "The URL (#{json['url']}) could not be retrieved."
+        when 'cannot-retrieve:http-redirect-limit'
+          raise RedirectionLimitError, "The URL (#{json['url']}) could not be retrieved, as it reached a redirect limit."
+        when 'page-is-not-html'
+          raise PageIsNotValidHtmlError, "The page at #{json['url']} is not valid HTML!"
+        when 'content-exceeds-size-limit'
+          raise ContentExceedsMaxLimitError, "The page at #{json['url']} is larger than 600KB!"
+        when 'invalid-html'
+          raise InvalidHtmlError, "The HTML sent was invalid!"
+        else
+          raise UnknownError, "Got an unknown error: #{json['statusInfo']}"
+        end
+      end
+    end
+  end
+end

data/lib/alchemy_api/categorization.rb

@@ -0,0 +1,47 @@
+module AlchemyApi
+  Category = Struct.new(:url, :name, :score)
+
+  class Categorization < Base
+    post(:get_categorization_from_text) do |text, *args|
+      options = args.first || {}
+      uri "#{AlchemyApi.base_text_uri}/TextGetCategory"
+      params :text => text,
+             :url => options[:url] || ''
+      handler do |response|
+        AlchemyApi::Categorization.get_categorization_handler(response)
+      end
+    end
+
+    post(:get_categorization_from_url) do |url, *args|
+      options = args.first || {}
+      uri "#{AlchemyApi.base_uri}/URLGetCategory"
+      params :url => url,
+             :cquery => options[:cquery] || '',
+             :sourceText => options[:source_text] || 'cleaned_or_raw',
+             :xpath => options[:xpath] || ''
+      handler do |response|
+        AlchemyApi::Categorization.get_categorization_handler(response)
+      end
+    end
+
+
+    post(:get_categorization_from_html) do |html, *args|
+      options = args.first || {}
+      uri "#{AlchemyApi.base_html_uri}/HTMLGetCategory"
+      params :html => html,
+             :url => options[:url] || '',
+             :cquery => options[:cquery] || '',
+             :sourceText => options[:source_text] || 'cleaned_or_raw',
+             :xpath => options[:xpath] || ''
+      handler do |response|
+        AlchemyApi::Categorization.get_categorization_handler(response)
+      end
+    end
+
+    def self.get_categorization_handler(response)
+      json = get_json(response)
+      Category.new(json['url'], json['category'],
+                   json['score'].to_f)
+    end
+  end
+end

data/lib/alchemy_api/language_detection.rb

@@ -0,0 +1,51 @@
+module AlchemyApi
+  LanguageResult = Struct.new(:url, :language, :iso_639_1, :iso_639_2,
+                              :iso_639_3, :ethnologue_url,
+                              :native_speakers, :wikipedia_url)
+
+  class LanguageDetection < Base
+    post(:get_language_from_url) do |url, *args|
+      options = args.first || {}
+      uri "#{AlchemyApi.base_uri}/URLGetLanguage"
+      params :url => url,
+             :sourceText => options[:source_text] || 'cleaned_or_raw',
+             :cquery => options[:cquery],
+             :xpath => options[:xpath]
+      handler do |response|
+        AlchemyApi::LanguageDetection.get_language_handler(response)
+      end
+    end
+
+    post(:get_language_from_html) do |html, *args|
+      options = args.first || {}
+      uri "#{AlchemyApi.base_html_uri}/HTMLGetLanguage"
+      params :html => html,
+             :url => options[:url],
+             :sourceText => options[:source_text] || 'cleaned_or_raw',
+             :cquery => options[:cquery],
+             :xpath => options[:xpath]
+      handler do |response|
+        AlchemyApi::LanguageDetection.get_language_handler(response)
+      end
+    end
+
+    post(:get_language_from_text) do |text, *args|
+      options = args.first || {}
+      uri "#{AlchemyApi.base_text_uri}/TextGetLanguage"
+      params :text => text,
+             :url => options[:url]
+      handler do |response|
+        AlchemyApi::LanguageDetection.get_language_handler(response)
+      end
+    end
+
+    def self.get_language_handler(response)
+      json = get_json(response)
+      LanguageResult.new(json['url'], json['language'], json['iso-639-1'],
+                         json['iso-639-2'], json['iso-639-3'],
+                         json['ethnologue'],
+                         json['native-speakers'],
+                         json['wikipedia'])
+    end
+  end
+end

data/lib/alchemy_api/term_extraction.rb

@@ -0,0 +1,56 @@
+module AlchemyApi
+  TermExtractionResult = Struct.new(:keywords, :language, :url, :source_text)
+  Keyword = Struct.new(:text, :relevance)
+
+  class TermExtraction < Base
+    post(:get_ranked_keywords_from_html) do |html, *args|
+      options = args.first || {}
+      uri "#{AlchemyApi.base_html_uri}/HTMLGetRankedKeywords"
+      params :html => html,
+             :url => options[:url],
+             :maxRetrieve => options[:max_retrieve] || 10,
+             :showSourceText => options[:show_source_text] ? 1 : 0,
+             :sourceText => options[:source_text] || 'cleaned_or_raw',
+             :cquery => options[:cquery],
+             :xpath => options[:xpath]
+      handler do |response|
+        AlchemyApi::TermExtraction.get_ranked_keywords_handler(response)
+      end
+    end
+
+    post(:get_ranked_keywords_from_url) do |url, *args|
+      options = args.first || {}
+      uri "#{AlchemyApi.base_uri}/URLGetRankedKeywords"
+      params :url => url,
+             :maxRetrieve => options[:max_retrieve] || 10,
+             :showSourceText => options[:show_source_text] ? 1 : 0,
+             :sourceText => options[:source_text] || 'cleaned_or_raw',
+             :cquery => options[:cquery],
+             :xpath => options[:xpath]
+      handler do |response|
+        AlchemyApi::TermExtraction.get_ranked_keywords_handler(response)
+      end
+    end
+
+    post(:get_ranked_keywords_from_text) do |text, *args|
+      options = args.first || {}
+      uri "#{AlchemyApi.base_text_uri}/TextGetRankedKeywords"
+      params :text => text,
+             :url => options[:url],
+             :maxRetrieve => options[:max_retrieve] || 10,
+             :showSourceText => options[:show_source_text] ? 1 : 0
+      handler do |response|
+        AlchemyApi::TermExtraction.get_ranked_keywords_handler(response)
+      end
+    end
+
+    def self.get_ranked_keywords_handler(response)
+      json = get_json(response)
+      keywords = json['keywords'].map do |kw|
+        Keyword.new(kw['text'], kw['relevance'].to_f)
+      end
+      TermExtractionResult.new(keywords, json['language'],
+                               json['url'], json['text'])
+    end
+  end
+end