yt_util 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4b3dfe4b4c3b37181be955ee39eb039d8385844d
+  data.tar.gz: 030dc138c1663339fc1b029689b7486d34fa07fa
+SHA512:
+  metadata.gz: a8fe781a3e22f5c1758e3171733ee85183204810563c6cf1602779add34265cc05d25b602101bd0efa60e9170d896e591f0547797f8942c823e0abd4e884a1b0
+  data.tar.gz: 478691e74b25f47ff4a9bfcec18703665c5432b9d071dde56d3e96473b0ab6a06cd48a394c9fb49b7535b5592b4633f159c9fc7b155945a5c0408d29d61f8a81

data/.gitignore

@@ -0,0 +1,15 @@
+.idea
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.gem
+*.so
+*.o
+*.a
+mkmf.log

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--warnings
+--require spec_helper

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in yt_util.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,45 @@
+PATH
+  remote: .
+  specs:
+    yt_util (0.0.1)
+      addressable (~> 2.3.6)
+      mechanize (~> 2.7.3)
+      nokogiri (~> 1.6.5)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.3.6)
+    domain_name (0.5.23)
+      unf (>= 0.0.5, < 1.0.0)
+    http-cookie (1.0.2)
+      domain_name (~> 0.5)
+    mechanize (2.7.3)
+      domain_name (~> 0.5, >= 0.5.1)
+      http-cookie (~> 1.0)
+      mime-types (~> 2.0)
+      net-http-digest_auth (~> 1.1, >= 1.1.1)
+      net-http-persistent (~> 2.5, >= 2.5.2)
+      nokogiri (~> 1.4)
+      ntlm-http (~> 0.1, >= 0.1.1)
+      webrobots (>= 0.0.9, < 0.2)
+    mime-types (2.4.3)
+    mini_portile (0.6.1)
+    net-http-digest_auth (1.4)
+    net-http-persistent (2.9.4)
+    nokogiri (1.6.5)
+      mini_portile (~> 0.6.0)
+    ntlm-http (0.1.1)
+    rake (10.4.2)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.6)
+    webrobots (0.1.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.6)
+  rake (~> 10.0)
+  yt_util!

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Daniel P. Clark
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,49 @@
+# Youtube Utility
+
+General purpose toolbox for working with Youtube.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'yt_util'
+```
+
+## Usage
+
+Usage notes will come in time.  Feel free to look at the source code as it's rather short.
+
+## NOTE
+> Any scraping features can break at anytime if Youtube chooses to change their page layout.
+
+## Contribute
+
+This project will become more and more complete over time.  Eventually the mapping of scraped data
+will be implemented intelligently instead of statically.  This way when Youtube changes their site
+the code will compensate for the change.  Feel free to contribute with this in mind.
+
+## LICENSE
+
+Copyright (c) 2014 Daniel P. Clark
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/lib/yt_util/scrape.rb

@@ -0,0 +1,75 @@
+module YtUtil
+  module Scrape
+
+    # Google search tips
+    # OR range etc https://support.google.com/websearch/answer/136861?hl=en
+    # punctuation: https://support.google.com/websearch/answer/2466433
+
+    def self.query(search, filters = "") # Returns an Array of Hash results
+      results = Mechanize.new.
+        tap { |i| i.follow_meta_refresh = true }.
+        get("https://www.youtube.com/results?search_query=#{Addressable::URI.parse(search).normalize + filters}")
+      parse_query(results.parser)
+    end
+
+    def self.video_statistics(video_code) # Video page hit.
+      result = Nokogiri::HTML(open("https://www.youtube.com/watch?v=#{video_code}"))
+      parse_video_page(result)
+    end
+
+    def self.user_statistics(username)
+      results = Mechanize.new.
+        tap { |i| i.follow_meta_refresh = true }.
+        get("https://www.youtube.com/user/#{username}/about")
+      parse_user(results.parser)
+    end
+
+    private
+    def self.parse_query(query_result)
+      query_result.css("ol.item-section > li")[1..-1].map do |result|
+        {
+          title: result.css("div:nth-child(1)").css("div:nth-child(2)").css("h3").text,
+          video: result.css("div:nth-child(1)").css("div:nth-child(2)").css("h3 > a").first[:href].dup.tap{|i|i.replace i[(i.index("=").to_i+1)..-1]},
+          views: Integer(String(/(\d+)/.match(result.css("div:nth-child(1)").css("div:nth-child(2)").css("li:nth-child(3)").text.gsub(",", "")))),
+          new: !!result.css("div:nth-child(1)").css("div:nth-child(2)").css("div:nth-child(4)").css("ul:nth-child(1)").text["New"],
+          hd: !!result.css("div:nth-child(1)").css("div:nth-child(2)").css("div:nth-child(4)").css("ul:nth-child(1)").text["HD"],
+          description: result.css("div:nth-child(1)").css("div:nth-child(2)").css("div:nth-child(3)").text,
+          length: result.css("div:nth-child(1)").css("div:nth-child(1)").css("a:nth-child(1)").css("span:nth-child(2)").text
+        }
+      end
+    end
+
+    def self.parse_video_page(query_result)
+      {
+        video: video_code,
+        user_name: query_result.css('a.yt-user-name').text,
+        description: query_result.css('p#eow-description').text,
+        category: query_result.css('li.watch-meta-item:nth-child(1)').css('ul:nth-child(2)').css('li:nth-child(1)').css('a:nth-child(1)').first[:href].try(:[], 1..-1),
+        # Views are rounded to nearest thousand.
+        views: String(/(\d+)/.match(query_result.css('div.watch-view-count').text.strip.gsub(',', ''))).to_i*1000,
+        # Likes are rounded to nearest thousand.
+        likes: String(/(\d+)/.match(query_result.css('button#watch-like').text.strip.gsub(',', ''))).to_i*1000,
+        # Dislikes are rounded to nearest thousand.
+        dislikes: String(/(\d+)/.match(query_result.css('button#watch-dislike').text.strip.gsub(',', ''))).to_i*1000,
+        published: query_result.css('strong.watch-time-text').text[13..-1],
+        license: query_result.css('li.watch-meta-item:nth-child(2)').text.gsub("\n", '').strip.tap {|i| i.replace i[i.index(' ').to_i..-1].strip}
+      }
+    end
+
+    def self.parse_user(query_result)
+      views_n_subs = query_result.css('.about-stats').
+        css('li').take(2).map{|i| i = i.text.strip; {
+        i.match(/[a-z]+/)[0] => i.match(/[\d,]+/)[0]}
+      }.inject(:update)
+
+      {
+        description: query_result.css('.about-description').css('p').text,
+        link: query_result.css('a[title="Google+"]')[0]["href"],
+        views: views_n_subs["views"],
+        subscribers: views_n_subs["subscribers"],
+        joined: query_result.css('.about-stats').css('.joined-date').text.strip
+      }
+    end
+
+  end
+end

data/lib/yt_util/url.rb

@@ -0,0 +1,52 @@
+module YtUtil
+  module URL
+    # https://developers.google.com/youtube/player_parameters
+    # Recommended included options for embedded video
+    # {:embed => true, :autoplay => true, :origin => request.env["REQUEST_URI"]}
+    def self.generate(video_code, options = {})
+      options.default = false
+      parameters = "?"
+      parameters.define_singleton_method(:ingest) {|item| self.<<("&") unless self[-1]["?"]; self.replace self.<<(item) }
+
+      parameters.ingest("autoplay=1") if options[:autoplay]
+      if options[:showplay]
+        parameters.ingest("autohide=2")
+      elsif options[:autohide]
+        parameters.ingest("autohide=1")
+      elsif options[:nohide]
+        parameters.ingest("autohide=0")
+      end
+      parameters.ingest("cc_load_policy=1") if options[:cc]
+      parameters.ingest("color=white") if options[:color] and !options[:modestbranding]
+      parameters.ingest("controls=0") if options[:nocontrols]
+      parameters.ingest("disablekb=1") if options[:nokeyboard]
+      parameters.ingest("enablejsapi=1") if options[:jsapi]
+      parameters.ingest("end=#{options[:end]}") if options[:end]
+      parameters.ingest("fs=0") if options[:nofullscreen]
+      # hl
+      parameters.ingest("iv_load_policy=3") if options[:noanotate]
+      # list
+      # listType
+      parameters.ingest("loop=1") if options[:loop]
+      parameters.ingest("modestbranding=1") if options[:noytlogo]
+      parameters.ingest("origin=http://#{URI.parse(options[:origin]).host}") if options[:origin]
+      # playerapiid
+      # playlist
+      parameters.ingest("playsinline=1") if options[:playinline]
+      parameters.ingest("rel=0") if options[:norelated]
+      parameters.ingest("showinfo=0") if options[:hideinfo]
+      parameters.ingest("start=#{options[:start]}") if options[:start]
+      parameters.ingest("theme=light") if options[:lighttheme]
+
+      parameters=("") if parameters.length.==(1)
+
+      if options[:short]
+        "http://youtu.be/#{video_code}".<<(parameters)
+      elsif options[:embed]
+        "//www.youtube.com/embed/#{video_code}".<<(parameters)
+      else
+        "https://www.youtube.com/watch?v=#{video_code}".<<(parameters.gsub("?","&"))
+      end
+    end
+  end
+end

data/lib/yt_util/version.rb

@@ -0,0 +1,3 @@
+module YtUtil
+  VERSION = "0.0.1"
+end

data/lib/yt_util.rb

@@ -0,0 +1,10 @@
+require "addressable/uri"
+require "open-uri"
+require "nokogiri"
+require "mechanize"
+require "yt_util/version"
+require "yt_util/scrape"
+require "yt_util/url"
+
+module YtUtil
+end

data/spec/spec_helper.rb

@@ -0,0 +1,78 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# The generated `.rspec` file contains `--require spec_helper` which will cause this
+# file to always be loaded, without a need to explicitly require it in any files.
+#
+# Given that it is always loaded, you are encouraged to keep this file as
+# light-weight as possible. Requiring heavyweight dependencies from this file
+# will add to the boot time of your test suite on EVERY test run, even for an
+# individual file that may not need all of that loaded. Instead, make a
+# separate helper file that requires this one and then use it only in the specs
+# that actually need it.
+#
+# The `.rspec` file also contains a few flags that are not defaults but that
+# users commonly want.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+  # These two settings work together to allow you to limit a spec run
+  # to individual examples or groups you care about by tagging them with
+  # `:focus` metadata. When nothing is tagged with `:focus`, all examples
+  # get run.
+  config.filter_run :focus
+  config.run_all_when_everything_filtered = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = 'doc'
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # Enable only the newer, non-monkey-patching expect syntax.
+    # For more details, see:
+    #   - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax
+    expectations.syntax = :expect
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Enable only the newer, non-monkey-patching expect syntax.
+    # For more details, see:
+    #   - http://teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+    mocks.syntax = :expect
+
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended.
+    mocks.verify_partial_doubles = true
+  end
+=end
+end

data/yt_util.gemspec

@@ -0,0 +1,28 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'yt_util/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "yt_util"
+  spec.version       = YtUtil::VERSION
+  spec.authors       = ["Daniel P. Clark"]
+  spec.email         = ["6ftdan@gmail.com"]
+  spec.summary       = %q{Youtube Utility.}
+  spec.description   = %q{A toolbox for Youtube.}
+  spec.homepage      = "http://github.com/danielpclark/yt_util"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.required_ruby_version = '~> 2.0'
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_runtime_dependency "nokogiri", "~> 1.6"
+  spec.add_runtime_dependency "mechanize", "~> 2.7"
+  spec.add_runtime_dependency "addressable", "~> 2.3"
+end

metadata

@@ -0,0 +1,128 @@
+--- !ruby/object:Gem::Specification
+name: yt_util
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Daniel P. Clark
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-12-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+description: A toolbox for Youtube.
+email:
+- 6ftdan@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/yt_util.rb
+- lib/yt_util/scrape.rb
+- lib/yt_util/url.rb
+- lib/yt_util/version.rb
+- spec/spec_helper.rb
+- yt_util.gemspec
+homepage: http://github.com/danielpclark/yt_util
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.3.0
+signing_key: 
+specification_version: 4
+summary: Youtube Utility.
+test_files:
+- spec/spec_helper.rb