yt_util 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 4b3dfe4b4c3b37181be955ee39eb039d8385844d
4
+ data.tar.gz: 030dc138c1663339fc1b029689b7486d34fa07fa
5
+ SHA512:
6
+ metadata.gz: a8fe781a3e22f5c1758e3171733ee85183204810563c6cf1602779add34265cc05d25b602101bd0efa60e9170d896e591f0547797f8942c823e0abd4e884a1b0
7
+ data.tar.gz: 478691e74b25f47ff4a9bfcec18703665c5432b9d071dde56d3e96473b0ab6a06cd48a394c9fb49b7535b5592b4633f159c9fc7b155945a5c0408d29d61f8a81
data/.gitignore ADDED
@@ -0,0 +1,15 @@
1
+ .idea
2
+ /.bundle/
3
+ /.yardoc
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
10
+ *.bundle
11
+ *.gem
12
+ *.so
13
+ *.o
14
+ *.a
15
+ mkmf.log
data/.rspec ADDED
@@ -0,0 +1,3 @@
1
+ --color
2
+ --warnings
3
+ --require spec_helper
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in yt_util.gemspec
4
+ gemspec
data/Gemfile.lock ADDED
@@ -0,0 +1,45 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ yt_util (0.0.1)
5
+ addressable (~> 2.3.6)
6
+ mechanize (~> 2.7.3)
7
+ nokogiri (~> 1.6.5)
8
+
9
+ GEM
10
+ remote: https://rubygems.org/
11
+ specs:
12
+ addressable (2.3.6)
13
+ domain_name (0.5.23)
14
+ unf (>= 0.0.5, < 1.0.0)
15
+ http-cookie (1.0.2)
16
+ domain_name (~> 0.5)
17
+ mechanize (2.7.3)
18
+ domain_name (~> 0.5, >= 0.5.1)
19
+ http-cookie (~> 1.0)
20
+ mime-types (~> 2.0)
21
+ net-http-digest_auth (~> 1.1, >= 1.1.1)
22
+ net-http-persistent (~> 2.5, >= 2.5.2)
23
+ nokogiri (~> 1.4)
24
+ ntlm-http (~> 0.1, >= 0.1.1)
25
+ webrobots (>= 0.0.9, < 0.2)
26
+ mime-types (2.4.3)
27
+ mini_portile (0.6.1)
28
+ net-http-digest_auth (1.4)
29
+ net-http-persistent (2.9.4)
30
+ nokogiri (1.6.5)
31
+ mini_portile (~> 0.6.0)
32
+ ntlm-http (0.1.1)
33
+ rake (10.4.2)
34
+ unf (0.1.4)
35
+ unf_ext
36
+ unf_ext (0.0.6)
37
+ webrobots (0.1.1)
38
+
39
+ PLATFORMS
40
+ ruby
41
+
42
+ DEPENDENCIES
43
+ bundler (~> 1.6)
44
+ rake (~> 10.0)
45
+ yt_util!
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014 Daniel P. Clark
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,49 @@
1
+ # Youtube Utility
2
+
3
+ General purpose toolbox for working with Youtube.
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ ```ruby
10
+ gem 'yt_util'
11
+ ```
12
+
13
+ ## Usage
14
+
15
+ Usage notes will come in time. Feel free to look at the source code as it's rather short.
16
+
17
+ ## NOTE
18
+ > Any scraping features can break at anytime if Youtube chooses to change their page layout.
19
+
20
+ ## Contribute
21
+
22
+ This project will become more and more complete over time. Eventually the mapping of scraped data
23
+ will be implemented intelligently instead of statically. This way when Youtube changes their site
24
+ the code will compensate for the change. Feel free to contribute with this in mind.
25
+
26
+ ## LICENSE
27
+
28
+ Copyright (c) 2014 Daniel P. Clark
29
+
30
+ MIT License
31
+
32
+ Permission is hereby granted, free of charge, to any person obtaining
33
+ a copy of this software and associated documentation files (the
34
+ "Software"), to deal in the Software without restriction, including
35
+ without limitation the rights to use, copy, modify, merge, publish,
36
+ distribute, sublicense, and/or sell copies of the Software, and to
37
+ permit persons to whom the Software is furnished to do so, subject to
38
+ the following conditions:
39
+
40
+ The above copyright notice and this permission notice shall be
41
+ included in all copies or substantial portions of the Software.
42
+
43
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
44
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
45
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
46
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
47
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
48
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
49
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/Rakefile ADDED
@@ -0,0 +1,2 @@
1
+ require "bundler/gem_tasks"
2
+
@@ -0,0 +1,75 @@
1
+ module YtUtil
2
+ module Scrape
3
+
4
+ # Google search tips
5
+ # OR range etc https://support.google.com/websearch/answer/136861?hl=en
6
+ # punctuation: https://support.google.com/websearch/answer/2466433
7
+
8
+ def self.query(search, filters = "") # Returns an Array of Hash results
9
+ results = Mechanize.new.
10
+ tap { |i| i.follow_meta_refresh = true }.
11
+ get("https://www.youtube.com/results?search_query=#{Addressable::URI.parse(search).normalize + filters}")
12
+ parse_query(results.parser)
13
+ end
14
+
15
+ def self.video_statistics(video_code) # Video page hit.
16
+ result = Nokogiri::HTML(open("https://www.youtube.com/watch?v=#{video_code}"))
17
+ parse_video_page(result)
18
+ end
19
+
20
+ def self.user_statistics(username)
21
+ results = Mechanize.new.
22
+ tap { |i| i.follow_meta_refresh = true }.
23
+ get("https://www.youtube.com/user/#{username}/about")
24
+ parse_user(results.parser)
25
+ end
26
+
27
+ private
28
+ def self.parse_query(query_result)
29
+ query_result.css("ol.item-section > li")[1..-1].map do |result|
30
+ {
31
+ title: result.css("div:nth-child(1)").css("div:nth-child(2)").css("h3").text,
32
+ video: result.css("div:nth-child(1)").css("div:nth-child(2)").css("h3 > a").first[:href].dup.tap{|i|i.replace i[(i.index("=").to_i+1)..-1]},
33
+ views: Integer(String(/(\d+)/.match(result.css("div:nth-child(1)").css("div:nth-child(2)").css("li:nth-child(3)").text.gsub(",", "")))),
34
+ new: !!result.css("div:nth-child(1)").css("div:nth-child(2)").css("div:nth-child(4)").css("ul:nth-child(1)").text["New"],
35
+ hd: !!result.css("div:nth-child(1)").css("div:nth-child(2)").css("div:nth-child(4)").css("ul:nth-child(1)").text["HD"],
36
+ description: result.css("div:nth-child(1)").css("div:nth-child(2)").css("div:nth-child(3)").text,
37
+ length: result.css("div:nth-child(1)").css("div:nth-child(1)").css("a:nth-child(1)").css("span:nth-child(2)").text
38
+ }
39
+ end
40
+ end
41
+
42
+ def self.parse_video_page(query_result)
43
+ {
44
+ video: video_code,
45
+ user_name: query_result.css('a.yt-user-name').text,
46
+ description: query_result.css('p#eow-description').text,
47
+ category: query_result.css('li.watch-meta-item:nth-child(1)').css('ul:nth-child(2)').css('li:nth-child(1)').css('a:nth-child(1)').first[:href].try(:[], 1..-1),
48
+ # Views are rounded to nearest thousand.
49
+ views: String(/(\d+)/.match(query_result.css('div.watch-view-count').text.strip.gsub(',', ''))).to_i*1000,
50
+ # Likes are rounded to nearest thousand.
51
+ likes: String(/(\d+)/.match(query_result.css('button#watch-like').text.strip.gsub(',', ''))).to_i*1000,
52
+ # Dislikes are rounded to nearest thousand.
53
+ dislikes: String(/(\d+)/.match(query_result.css('button#watch-dislike').text.strip.gsub(',', ''))).to_i*1000,
54
+ published: query_result.css('strong.watch-time-text').text[13..-1],
55
+ license: query_result.css('li.watch-meta-item:nth-child(2)').text.gsub("\n", '').strip.tap {|i| i.replace i[i.index(' ').to_i..-1].strip}
56
+ }
57
+ end
58
+
59
+ def self.parse_user(query_result)
60
+ views_n_subs = query_result.css('.about-stats').
61
+ css('li').take(2).map{|i| i = i.text.strip; {
62
+ i.match(/[a-z]+/)[0] => i.match(/[\d,]+/)[0]}
63
+ }.inject(:update)
64
+
65
+ {
66
+ description: query_result.css('.about-description').css('p').text,
67
+ link: query_result.css('a[title="Google+"]')[0]["href"],
68
+ views: views_n_subs["views"],
69
+ subscribers: views_n_subs["subscribers"],
70
+ joined: query_result.css('.about-stats').css('.joined-date').text.strip
71
+ }
72
+ end
73
+
74
+ end
75
+ end
@@ -0,0 +1,52 @@
1
+ module YtUtil
2
+ module URL
3
+ # https://developers.google.com/youtube/player_parameters
4
+ # Recommended included options for embedded video
5
+ # {:embed => true, :autoplay => true, :origin => request.env["REQUEST_URI"]}
6
+ def self.generate(video_code, options = {})
7
+ options.default = false
8
+ parameters = "?"
9
+ parameters.define_singleton_method(:ingest) {|item| self.<<("&") unless self[-1]["?"]; self.replace self.<<(item) }
10
+
11
+ parameters.ingest("autoplay=1") if options[:autoplay]
12
+ if options[:showplay]
13
+ parameters.ingest("autohide=2")
14
+ elsif options[:autohide]
15
+ parameters.ingest("autohide=1")
16
+ elsif options[:nohide]
17
+ parameters.ingest("autohide=0")
18
+ end
19
+ parameters.ingest("cc_load_policy=1") if options[:cc]
20
+ parameters.ingest("color=white") if options[:color] and !options[:modestbranding]
21
+ parameters.ingest("controls=0") if options[:nocontrols]
22
+ parameters.ingest("disablekb=1") if options[:nokeyboard]
23
+ parameters.ingest("enablejsapi=1") if options[:jsapi]
24
+ parameters.ingest("end=#{options[:end]}") if options[:end]
25
+ parameters.ingest("fs=0") if options[:nofullscreen]
26
+ # hl
27
+ parameters.ingest("iv_load_policy=3") if options[:noanotate]
28
+ # list
29
+ # listType
30
+ parameters.ingest("loop=1") if options[:loop]
31
+ parameters.ingest("modestbranding=1") if options[:noytlogo]
32
+ parameters.ingest("origin=http://#{URI.parse(options[:origin]).host}") if options[:origin]
33
+ # playerapiid
34
+ # playlist
35
+ parameters.ingest("playsinline=1") if options[:playinline]
36
+ parameters.ingest("rel=0") if options[:norelated]
37
+ parameters.ingest("showinfo=0") if options[:hideinfo]
38
+ parameters.ingest("start=#{options[:start]}") if options[:start]
39
+ parameters.ingest("theme=light") if options[:lighttheme]
40
+
41
+ parameters=("") if parameters.length.==(1)
42
+
43
+ if options[:short]
44
+ "http://youtu.be/#{video_code}".<<(parameters)
45
+ elsif options[:embed]
46
+ "//www.youtube.com/embed/#{video_code}".<<(parameters)
47
+ else
48
+ "https://www.youtube.com/watch?v=#{video_code}".<<(parameters.gsub("?","&"))
49
+ end
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,3 @@
1
+ module YtUtil
2
+ VERSION = "0.0.1"
3
+ end
data/lib/yt_util.rb ADDED
@@ -0,0 +1,10 @@
1
+ require "addressable/uri"
2
+ require "open-uri"
3
+ require "nokogiri"
4
+ require "mechanize"
5
+ require "yt_util/version"
6
+ require "yt_util/scrape"
7
+ require "yt_util/url"
8
+
9
+ module YtUtil
10
+ end
@@ -0,0 +1,78 @@
1
+ # This file was generated by the `rspec --init` command. Conventionally, all
2
+ # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
3
+ # The generated `.rspec` file contains `--require spec_helper` which will cause this
4
+ # file to always be loaded, without a need to explicitly require it in any files.
5
+ #
6
+ # Given that it is always loaded, you are encouraged to keep this file as
7
+ # light-weight as possible. Requiring heavyweight dependencies from this file
8
+ # will add to the boot time of your test suite on EVERY test run, even for an
9
+ # individual file that may not need all of that loaded. Instead, make a
10
+ # separate helper file that requires this one and then use it only in the specs
11
+ # that actually need it.
12
+ #
13
+ # The `.rspec` file also contains a few flags that are not defaults but that
14
+ # users commonly want.
15
+ #
16
+ # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
17
+ RSpec.configure do |config|
18
+ # The settings below are suggested to provide a good initial experience
19
+ # with RSpec, but feel free to customize to your heart's content.
20
+ =begin
21
+ # These two settings work together to allow you to limit a spec run
22
+ # to individual examples or groups you care about by tagging them with
23
+ # `:focus` metadata. When nothing is tagged with `:focus`, all examples
24
+ # get run.
25
+ config.filter_run :focus
26
+ config.run_all_when_everything_filtered = true
27
+
28
+ # Many RSpec users commonly either run the entire suite or an individual
29
+ # file, and it's useful to allow more verbose output when running an
30
+ # individual spec file.
31
+ if config.files_to_run.one?
32
+ # Use the documentation formatter for detailed output,
33
+ # unless a formatter has already been configured
34
+ # (e.g. via a command-line flag).
35
+ config.default_formatter = 'doc'
36
+ end
37
+
38
+ # Print the 10 slowest examples and example groups at the
39
+ # end of the spec run, to help surface which specs are running
40
+ # particularly slow.
41
+ config.profile_examples = 10
42
+
43
+ # Run specs in random order to surface order dependencies. If you find an
44
+ # order dependency and want to debug it, you can fix the order by providing
45
+ # the seed, which is printed after each run.
46
+ # --seed 1234
47
+ config.order = :random
48
+
49
+ # Seed global randomization in this process using the `--seed` CLI option.
50
+ # Setting this allows you to use `--seed` to deterministically reproduce
51
+ # test failures related to randomization by passing the same `--seed` value
52
+ # as the one that triggered the failure.
53
+ Kernel.srand config.seed
54
+
55
+ # rspec-expectations config goes here. You can use an alternate
56
+ # assertion/expectation library such as wrong or the stdlib/minitest
57
+ # assertions if you prefer.
58
+ config.expect_with :rspec do |expectations|
59
+ # Enable only the newer, non-monkey-patching expect syntax.
60
+ # For more details, see:
61
+ # - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax
62
+ expectations.syntax = :expect
63
+ end
64
+
65
+ # rspec-mocks config goes here. You can use an alternate test double
66
+ # library (such as bogus or mocha) by changing the `mock_with` option here.
67
+ config.mock_with :rspec do |mocks|
68
+ # Enable only the newer, non-monkey-patching expect syntax.
69
+ # For more details, see:
70
+ # - http://teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
71
+ mocks.syntax = :expect
72
+
73
+ # Prevents you from mocking or stubbing a method that does not exist on
74
+ # a real object. This is generally recommended.
75
+ mocks.verify_partial_doubles = true
76
+ end
77
+ =end
78
+ end
data/yt_util.gemspec ADDED
@@ -0,0 +1,28 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'yt_util/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "yt_util"
8
+ spec.version = YtUtil::VERSION
9
+ spec.authors = ["Daniel P. Clark"]
10
+ spec.email = ["6ftdan@gmail.com"]
11
+ spec.summary = %q{Youtube Utility.}
12
+ spec.description = %q{A toolbox for Youtube.}
13
+ spec.homepage = "http://github.com/danielpclark/yt_util"
14
+ spec.license = "MIT"
15
+
16
+ spec.files = `git ls-files -z`.split("\x0")
17
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
19
+ spec.require_paths = ["lib"]
20
+
21
+ spec.required_ruby_version = '~> 2.0'
22
+
23
+ spec.add_development_dependency "bundler", "~> 1.6"
24
+ spec.add_development_dependency "rake", "~> 10.0"
25
+ spec.add_runtime_dependency "nokogiri", "~> 1.6"
26
+ spec.add_runtime_dependency "mechanize", "~> 2.7"
27
+ spec.add_runtime_dependency "addressable", "~> 2.3"
28
+ end
metadata ADDED
@@ -0,0 +1,128 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: yt_util
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Daniel P. Clark
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-12-23 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.6'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.6'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '10.0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '10.0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: nokogiri
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '1.6'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '1.6'
55
+ - !ruby/object:Gem::Dependency
56
+ name: mechanize
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '2.7'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '2.7'
69
+ - !ruby/object:Gem::Dependency
70
+ name: addressable
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '2.3'
76
+ type: :runtime
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '2.3'
83
+ description: A toolbox for Youtube.
84
+ email:
85
+ - 6ftdan@gmail.com
86
+ executables: []
87
+ extensions: []
88
+ extra_rdoc_files: []
89
+ files:
90
+ - ".gitignore"
91
+ - ".rspec"
92
+ - Gemfile
93
+ - Gemfile.lock
94
+ - LICENSE.txt
95
+ - README.md
96
+ - Rakefile
97
+ - lib/yt_util.rb
98
+ - lib/yt_util/scrape.rb
99
+ - lib/yt_util/url.rb
100
+ - lib/yt_util/version.rb
101
+ - spec/spec_helper.rb
102
+ - yt_util.gemspec
103
+ homepage: http://github.com/danielpclark/yt_util
104
+ licenses:
105
+ - MIT
106
+ metadata: {}
107
+ post_install_message:
108
+ rdoc_options: []
109
+ require_paths:
110
+ - lib
111
+ required_ruby_version: !ruby/object:Gem::Requirement
112
+ requirements:
113
+ - - "~>"
114
+ - !ruby/object:Gem::Version
115
+ version: '2.0'
116
+ required_rubygems_version: !ruby/object:Gem::Requirement
117
+ requirements:
118
+ - - ">="
119
+ - !ruby/object:Gem::Version
120
+ version: '0'
121
+ requirements: []
122
+ rubyforge_project:
123
+ rubygems_version: 2.3.0
124
+ signing_key:
125
+ specification_version: 4
126
+ summary: Youtube Utility.
127
+ test_files:
128
+ - spec/spec_helper.rb