yql_simple 0.0.2

data/.gitignore

@@ -0,0 +1,6 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+doc
+ignore

data/.travis.yml

@@ -0,0 +1,16 @@
+# Configuration file for travis-ci.org
+
+# Define a custom build script
+# You can specify the main build command to run instead of just rake:
+script: "bundle exec rspec spec"
+
+# specify which ruby version to use
+rvm:
+  - 1.8.7
+  - 1.9.2
+  - jruby
+  # - 1.8.6 not working
+
+# turn off all notifications
+notifications:
+  disabled: true

data/Gemfile

@@ -0,0 +1,3 @@
+source :rubygems
+
+gemspec

data/README.md

@@ -0,0 +1,105 @@
+Simple Ruby gem for YQL
+-----------------------
+-----------------------
+
+This is currently more an experiment of ruby gem building than a real ruby gem!!! The gem is usable but still may not be useful for anybody other than myself! Considered yourself at danger if you use it :)
+
+This is my first implementation of a simple wrapper for [YQL](http://developer.yahoo.com/yql/) queries. It does not do anything fancy. Instead it just sends an HTTP request to YQL with the given query and returns the results as either a ruby Hash or Nokogiri::XML::Document.
+
+I am using [bundler](http://gembundler.com) for the fetching the dependencies, [RSpec](http://relishapp.com/rspec) for the testing, and [RDoc](http://rdoc.sourceforge.net/doc/index.html) for code documentation.
+
+
+Useful Reads for Gem Building
+-----------------------------
+
+* [Developing a RubyGem using Bundler](https://github.com/radar/guides/blob/master/gem-development.md)
+* [Making Ruby Gems](http://timelessrepo.com/making-ruby-gems)
+* [Using .gemspecs as Intended](http://yehudakatz.com/2010/04/02/using-gemspecs-as-intended/)
+
+
+Continuous Integration
+-----------------------------
+
+Shows the current built status of this gem, as reported by [Travis CI](http://travis-ci.org/)
+
+[![Build Status](https://secure.travis-ci.org/spier/gem_yql_simple.png)](http://travis-ci.org/spier/gem_yql_simple)
+
+
+Installation
+-----------------------------
+
+The gem is not released yet, so it is not on [http://rubygems.org](http://rubygems.org) or similar.
+If you really want to try it, what I would not recommend, then you have to build it yourself from source.
+
+If you are using bundler you can do the following to install directly from the source:
+
+	gem "yql_simple", :git => "git://github.com/spier/gem_yql_simple.git"
+	
+Usage
+-----------------------------
+
+Example of doing a twitter search, using the yql_simple gem. You can also find this exaple in the *examples* folder.
+
+	#!/usr/bin/env ruby
+	require "rubygems"
+	require "bundler/setup"
+
+	require "yql_simple"
+
+	query_string = 'SELECT * FROM twitter.search WHERE q="#ruby" LIMIT 5'
+	response = YqlSimple.query(query_string)
+
+	response["query"]["results"]["results"].each do |tweet| 
+	  date = Time.parse(tweet["created_at"]).strftime("%Y-%m-%d %H:%M")  
+	  puts "[#{date}] #{tweet["from_user"]}: #{tweet["text"]}"
+	end
+		
+This will output something like this:
+
+	[2011-05-15 13:12] eyoosuf: Rails and Django for Quick thinking Developers, PHP for Lazy Developers #PHP #RUBY #ROR #PYTHON #Rails #Django
+	[2011-05-15 12:57] rudkovsky: Looking for a job! #rails #ruby #job
+	[2011-05-15 12:48] IndianGuru: Podcaster Miles Forrest interviewing RubyLearning;'s mentor Victor Goff this 18th #ruby
+	[2011-05-15 12:41] openairjoy: outdoor: http://openairjoy.com/05/09/woodstock-chimes-ruby-throated-hummingbird-windchime.html Woodstock Chimes R #Chimes #Hummingbird #Ruby
+	[2011-05-15 12:27] Elusive_Joe: I'm not entirely successfully installed gitorious at work. It seems this is the problem of #gitorious with #ruby 1.9.2
+	
+
+Releases
+-----------------------------
+
+0.0.2
+- added OAuth supported for 2-legged OAuth with YQL
+
+0.0.1 (unreleased)
+- added minimal RDoc documentation
+- RDoc will be generated with `rake rdoc` or when installing the gem
+- test installation via bundler and :git
+
+
+Notes for Contributors
+-----------------------------
+
+* build and install
+
+	(sudo) rake install
+
+* run the tests (this will check for a folder 'spec' and run rspec against all _spec files)
+
+	rake spec
+	
+* generate rdoc (the rdoc documentation will be generated in ./doc)
+
+	rake rdoc
+
+* get a list of rake tasks
+
+	rake -T
+	
+* building the gem
+	gem build yql_simple.gemspec
+	
+
+	
+Alternative Gems
+-----------------------------
+
+https://github.com/nas/yql

data/Rakefile

@@ -0,0 +1,33 @@
+require 'bundler'
+
+# get the rake tasks build, install, and release from Bundler
+Bundler::GemHelper.install_tasks
+require 'rspec/core/rake_task'
+
+# see example
+# https://github.com/twitter/twitter-text-rb/blob/master/Rakefile
+desc 'generate API documentation to doc/rdocs/index.html'
+require 'rdoc/task'
+
+Rake::RDocTask.new do |rd|
+  namespace :doc do
+    rd.main = "README.rdoc"
+    rd.rdoc_dir = 'doc'
+    rd.rdoc_files.include("README.rdoc", "lib/**/*.rb")
+
+    rd.options << '--inline-source'
+    rd.options << '--line-numbers'
+    rd.options << '--all'
+
+    # For the --diagram and --fileboxes options to work you need GraphViz otherwise, delete those.
+    # rd.options << '--fileboxes'
+    # rd.options << '--diagram'
+  end
+end
+
+# see http://duckpunching.com/rspec-2-0-rake-tasks
+desc "Run specs (tests)"
+RSpec::Core::RakeTask.new do |t|
+  t.pattern = "./spec/**/*_spec.rb" # don't need this, it's default.
+  # Put spec opts in a file named .rspec in root
+end

data/TODO.md

@@ -0,0 +1,21 @@
+- implement OAuth with YQL (will give the user more calls to YQL)
+- how to test the validity of the OAuth parameters?
+
+- add more test
+
+- make the desired node in the YQL response selectable e.g. query/results/item
+
+- add [YQL Streaming](http://developer.yahoo.com/yql/guide/yql-odt-streaming.html) 
+
+- how to properly raise exceptions?
+	https://github.com/outsidein/api-rb/blob/master/lib/outside_in/resource/base.rb#L46	
+
+- get a YQL application key here
+	https://developer.apps.yahoo.com/projects
+
+- externalize the env= URL parameter and allow the use of other customer YQL tables that are not part of the YQL community tables (they can already be used in the YQL query itself but not via an URL parameter)
+- enable switching SSL on/off
+- should the top level wrapper of YqlSimple be a class or a module?
+- handle YQL status codes (which indicate errors)
+- write some tests for YQL exceptions
+- currently the ./examples folder is added to the gem. not really good. how to exclude it?

data/examples/Gemfile

@@ -0,0 +1,3 @@
+source :rubygems
+
+gemspec

data/examples/gem_usage.rb

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+require "rubygems"
+require "bundler/setup"
+
+require "yql_simple"
+
+#
+# Example of doing a twitter search via YQL
+#
+
+query_string = 'SELECT * FROM twitter.search WHERE q="#ruby" LIMIT 5'
+response = YqlSimple.query(query_string)
+
+response["query"]["results"]["results"].each do |tweet| 
+  date = Time.parse(tweet["created_at"]).strftime("%Y-%m-%d %H:%M")  
+  puts "[#{date}] #{tweet["from_user"]}: #{tweet["text"]}"
+end

data/lib/yql_simple.rb

@@ -0,0 +1 @@
+require 'yql_simple/base'

data/lib/yql_simple/base.rb

@@ -0,0 +1,170 @@
+require 'open-uri'
+require 'pp'
+require 'cgi'
+
+# Load all required dependencies here manually.
+# No bundler usage, as this would make the gem dependent on bundler!
+
+# require 'json'
+# require 'nokogiri'
+# require 'curb'
+require 'httparty'
+require 'oauth'
+require 'json'
+require 'logger'
+
+module YqlSimple
+
+  class Client
+    include HTTParty
+    base_uri 'https://query.yahooapis.com/v1/public/yql'
+
+    # @@yql_api_url = "https://query.yahooapis.com/v1/public/yql"
+    @@env = "store://datatables.org/alltableswithkeys"
+
+    # attr_accessor :yql_api_url
+    # attr_accessor :env
+
+    #get
+    def self.env
+      @@env
+    end
+
+    #set
+    def self.env= (new_env)
+      @@env = new_env
+    end
+
+    # #get
+    # def self.yql_api_url
+    #   @@yql_api_url
+    # end
+    #
+    # #set
+    # def self.yql_api_url=(new_yql_api_url)
+    #   @@yql_api_url = new_yql_api_url
+    # end
+
+    def initialize()
+    # def initialize(apikey)
+      @log = Logger.new(STDOUT)
+      @log.level = Logger::WARN
+      # @apikey = apikey
+    end
+  end
+
+
+  class SimpleClient < Client
+    include HTTParty
+    base_uri 'https://query.yahooapis.com/v1/public/yql'
+
+    # Runs the given <b>yql_query</b> and returns the complete response.
+    # Depending on the <b>format</b> parameter the result is either a ruby Hash (parsed from JSON) or a Nokogiri::XML::Document. The default is 'json'.
+    # The <b>diagnostics</b parameter specifies whether or not to return query diagnostics. The default is 'false'.
+    #
+    def query(yql_query, format='json', diagnostics='false')
+      # TODO tried to make the necessary parameters more flexibel. failed for now :)
+      # format = (hash.nil? or hash[:format.to_s].nil?) ? 'json' : hash[:format.to_s]
+      # format = (hash && hash[:format]) || 'json'
+
+      @log.info "SimpleClient query()"
+
+      # try to run YQL query. If an error occures return an empty array
+      begin
+
+        #
+        # Httparty version
+        #
+        options = {
+          :query => {
+            :env => SimpleClient.env, #"store://datatables.org/alltableswithkeys",
+            :diagnostics => diagnostics,
+            :format => format,
+            :q => yql_query
+          }
+        }
+
+        yahoo_response = self.class.get('/', options)
+
+        #
+        # Curl Version
+        #
+
+        # # put together request URL
+        # full_request_url = "#{@@yql_api_url}?diagnostics=#{diagnostics}"
+        # full_request_url += "&#{@@env}"
+        # full_request_url += "&format=#{format}"
+        # full_request_url += "&q=#{CGI.escape(yql_query)}"
+
+        # request = Curl::Easy.http_get(full_request_url)
+        # request.perform
+        # # pp request.header_str
+        # if format == 'json'
+        #   yahoo_response = JSON.parse(request.body_str)
+        # elsif format == 'xml'
+        #   yahoo_response = Nokogiri::XML(request.body_str)
+        # end
+
+      rescue Exception => ex
+        puts ex
+      end
+
+      return yahoo_response
+    end
+
+  end
+
+
+  # from https://github.com/jnunemaker/twitter/blob/b9fcb86e9b5857940e290863474f8137fa899a49/lib/twitter/oauth.rb
+  class OAuthClient < Client
+    attr_reader :ctoken, :csecret, :consumer
+
+    def initialize(ctoken, csecret)
+      super()
+      @ctoken = ctoken
+      @csecret = csecret
+    end
+
+    def consumer
+      @consumer ||= OAuth::Consumer.new(@ctoken, @csecret, config())
+    end
+
+    def config
+      {
+        :site               => "https://api.login.yahoo.com",
+        :scheme             => :header,
+        :http_method        => :post,
+        :request_token_path => "/oauth/v2/get_request_token",
+        :access_token_path  => "/oauth/v2/get_access_token"
+      }
+    end
+
+    def access_token
+      @access_token ||= OAuth::AccessToken.new(consumer())
+    end
+
+    def query(yql_query, format='json', diagnostics='false')
+      @log.info "OAuthClient query()"
+
+      # TODO these options are not used yet!!!
+      options = {
+        :env => "store://datatables.org/alltableswithkeys",
+        :diagnostics => diagnostics,
+        :format => format,
+        :q => yql_query
+      }
+
+      # yahoo_response = self.class.get('/', options)
+      options_string = options.map{|k,v| "#{k}=#{CGI.escape(v)}"}.join("&")
+
+      response = access_token.get("http://query.yahooapis.com/v1/yql?#{options_string}")
+      # response = access_token.get("http://query.yahooapis.com/v1/yql",options)
+      return JSON[response.body]
+    end
+
+    def to_str
+      "ctoken: #{@ctoken}, csecret: #{@csecret}"
+    end
+  end
+
+end

data/lib/yql_simple/version.rb

@@ -0,0 +1,4 @@
+module YqlSimple
+  # versioning will follow semantic versioning: http://semver.org
+  VERSION = "0.0.2"
+end

data/spec/yql_simple_spec.rb

@@ -0,0 +1,123 @@
+# require all gems from Bundler (default and development)
+Bundler.require(:default, :development)
+
+# the gem itself is available by default
+# require 'yql_simple'
+
+# the number of core tables changes but it should be more than 100
+YQL_CORE_TABLE_COUNT = 100
+
+# my oauth credentials
+# TODO set up a dummy account here, as this will be public
+OAUTH_CONSUMER_KEY = 'dj0yJmk9V3hYVGNTeTNHQ3ZsJmQ9WVdrOWRIQllVRGxNTjJrbWNHbzlNVEV5TlRZd01qSTJNZy0tJnM9Y29uc3VtZXJzZWNyZXQmeD0wMg--'
+OAUTH_CONSUMER_SECRET = '4a183185ddc8bd6934f11116cf96895547d97508'
+
+# Tests
+# Docs: http://relishapp.com/rspec
+describe YqlSimple do
+
+  # create a new instance of SimpleClient
+  simple_client = YqlSimple::SimpleClient.new()
+
+  it "OAuth success" do
+    query_string = "SHOW TABLES"
+
+    # create client
+    oauth_client = YqlSimple::OAuthClient.new(OAUTH_CONSUMER_KEY,OAUTH_CONSUMER_SECRET)
+
+    response = oauth_client.query(query_string, "json", "true")
+
+    core_tables = response["query"]["results"]["table"].select{|entry| not entry.is_a?(Hash)}
+    core_tables.size.should >= YQL_CORE_TABLE_COUNT
+
+    # ----------
+
+    # all commits from the github feed should have an author key
+    query_string = 'SELECT * FROM feed WHERE url="https://github.com/spier/gem_yql_simple/commits/master.atom"'
+    response = oauth_client.query(query_string, 'json')
+    response["query"]["results"]["entry"][0].should have_key("author")
+  end
+
+  # it "OAuth failure" do
+  #   query_string = "SHOW TABLES"
+  #
+  #   # create client with wrong credentials
+  #   oauth_client = YqlSimple::OAuthClient.new("I_AM_WRONG","I_AM_WRONG")
+  #   puts oauth_client.to_str
+  #
+  #   lambda { oauth_client.query(query_string, "json", "true") }.should raise_error()
+  # end
+
+
+  it "test diagnostics parameter" do
+    query_string = "SHOW TABLES"
+
+    # diagnostics on
+    response = simple_client.query(query_string, "json", "true")
+    # pp response
+    response["query"].should have_key("diagnostics")
+
+    # diagnostics off
+    response = simple_client.query(query_string, "json", "false")
+    response["query"].should_not have_key("diagnostics")
+  end
+
+  # when using HTTParty the response is already a Ruby object
+
+  # it "test XML response" do
+  #   query_string = "SHOW TABLES"
+  #
+  #   response = YqlSimple.query(query_string, 'xml')
+  #
+  #   response.class.should be_a(Nokogiri::XML::Document.class)
+  # end
+
+  it "test JSON response" do
+    query_string = "SHOW TABLES"
+
+    response = simple_client.query(query_string, 'json')
+
+    response.class.should be_a(Hash.class)
+  end
+
+  it "test for YQL core tables" do
+    query_string = "SHOW TABLES"
+
+    response = simple_client.query(query_string, 'json')
+
+    core_tables = response["query"]["results"]["table"].select{|entry| not entry.is_a?(Hash)}
+    core_tables.size.should >= YQL_CORE_TABLE_COUNT
+  end
+
+  it "test some simple YQL calls" do
+    # should return a status code of 200
+    # query_string = 'SELECT * FROM data.headers WHERE url="https://github.com/spier/gem_yql_simple"'
+    # response = YqlSimple.query(query_string, 'json')
+    # pp response
+    # response["query"]["results"]["resources"]["status"].should == "200"
+
+    # all commits from the github feed should have an author key
+    query_string = 'SELECT * FROM feed WHERE url="https://github.com/spier/gem_yql_simple/commits/master.atom"'
+    response = simple_client.query(query_string, 'json')
+    response["query"]["results"]["entry"][0].should have_key("author")
+
+    # YQL queries to github are forbidden
+    expected_error_msg = 'Redirected to a robots.txt restricted URL: https://github.com/spier/gem_yql_simple'
+    query_string = 'SELECT * FROM html WHERE url="https://github.com/spier/gem_yql_simple"'
+    response = simple_client.query(query_string, 'json',"true")
+    response["query"]["diagnostics"]["url"]["error"].should == expected_error_msg
+  end
+
+  it "test variable access" do
+    # test set env
+    env = "something"
+    YqlSimple::SimpleClient.env = env
+    YqlSimple::SimpleClient.env.should == env
+
+    # # test set yql_api_url
+    # yql_api = "url_to_api"
+    # YqlSimple.yql_api_url = yql_api
+    # YqlSimple.yql_api_url.should == yql_api
+  end
+
+end

data/yql_simple.gemspec

@@ -0,0 +1,34 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "yql_simple/version"
+
+Gem::Specification.new do |s|
+  # general gem information
+  s.name        = "yql_simple"
+  s.version     = YqlSimple::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Sebastian Spier"]
+  s.email       = ["blog@airness.de"]
+  s.homepage    = "https://github.com/spier/gem_yql_simple"
+  s.description = s.summary = %q{Simple wrapper for calls to YQL}
+
+  s.rdoc_options << '--title' << '--main' << 'README.rdoc' << '--line-numbers' << '--inline-source'
+  #s.rubyforge_project = "yql_simple"
+
+  # all dependencies of this gem
+  # s.add_dependency "nokogiri"
+  s.add_dependency "json"
+  # s.add_dependency "curb"
+  s.add_dependency "httparty"
+  s.add_dependency "oauth"
+
+  # dependencies when developing this gem
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "rake"
+
+  # all files that will be added to this gem. git is used for this
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

metadata

@@ -0,0 +1,119 @@
+--- !ruby/object:Gem::Specification
+name: yql_simple
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+  prerelease: 
+platform: ruby
+authors:
+- Sebastian Spier
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-05-06 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: &2803540 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2803540
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: &2803330 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2803330
+- !ruby/object:Gem::Dependency
+  name: oauth
+  requirement: &2803120 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2803120
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &2802910 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *2802910
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &2802700 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *2802700
+description: Simple wrapper for calls to YQL
+email:
+- blog@airness.de
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- README.md
+- Rakefile
+- TODO.md
+- examples/Gemfile
+- examples/gem_usage.rb
+- lib/yql_simple.rb
+- lib/yql_simple/base.rb
+- lib/yql_simple/version.rb
+- spec/yql_simple_spec.rb
+- yql_simple.gemspec
+homepage: https://github.com/spier/gem_yql_simple
+licenses: []
+post_install_message: 
+rdoc_options:
+- --title
+- --main
+- README.rdoc
+- --line-numbers
+- --inline-source
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.6
+signing_key: 
+specification_version: 3
+summary: Simple wrapper for calls to YQL
+test_files:
+- spec/yql_simple_spec.rb