bulk_youtube 0.1.0 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.floo +3 -0
- data/.flooignore +8 -0
- data/lib/bulk_youtube.rb +6 -29
- data/lib/bulk_youtube/benchmark_testing.rb +140 -0
- data/lib/bulk_youtube/version.rb +1 -1
- data/spec/benchmark_spec.rb +17 -0
- data/spec/full_page_playlist_spec.rb +9 -2
- metadata +7 -2
data/.floo
ADDED
data/.flooignore
ADDED
data/lib/bulk_youtube.rb
CHANGED
@@ -2,6 +2,7 @@ require "bulk_youtube/version"
|
|
2
2
|
require "bulk_youtube/globals"
|
3
3
|
require 'mechanize'
|
4
4
|
require 'nokogiri'
|
5
|
+
require 'benchmark'
|
5
6
|
|
6
7
|
|
7
8
|
module BulkYoutube
|
@@ -18,22 +19,6 @@ module BulkYoutube
|
|
18
19
|
end
|
19
20
|
|
20
21
|
|
21
|
-
# def you_convert
|
22
|
-
# retry_count = 0
|
23
|
-
# begin
|
24
|
-
# send_files(@link_arr)
|
25
|
-
# rescue Net::HTTP::Persistent::Error => e
|
26
|
-
# raise unless e.message =~ /too man connection resets/
|
27
|
-
# if retry_count > @max_retries
|
28
|
-
# puts "**** WARN: Mechanize retried connection reset #{@max_retries} times"
|
29
|
-
# raise
|
30
|
-
# end
|
31
|
-
# retry_count +=1
|
32
|
-
# Mechanize::HTTP::Agent.http.shutdown
|
33
|
-
# retry
|
34
|
-
# end
|
35
|
-
# end
|
36
|
-
|
37
22
|
def you_convert
|
38
23
|
begin
|
39
24
|
send_files(@link_arr)
|
@@ -66,7 +51,6 @@ module BulkYoutube
|
|
66
51
|
|
67
52
|
|
68
53
|
|
69
|
-
|
70
54
|
private
|
71
55
|
|
72
56
|
def noko(path)
|
@@ -74,7 +58,7 @@ module BulkYoutube
|
|
74
58
|
end
|
75
59
|
|
76
60
|
def mec_get(path)
|
77
|
-
Mechanize.new.get(path)
|
61
|
+
Mechanize.new.get(path)
|
78
62
|
end
|
79
63
|
|
80
64
|
# returns itself
|
@@ -85,27 +69,20 @@ module BulkYoutube
|
|
85
69
|
|
86
70
|
if word.nil?
|
87
71
|
|
88
|
-
path.
|
72
|
+
path.links_with(:href => /watch\?v=/).each do |link|
|
89
73
|
break if count > @max_downloads
|
90
|
-
youtube = link.click.uri.to_s
|
91
74
|
|
92
|
-
# next if link.text.include?('Play')
|
93
|
-
|
94
|
-
if youtube.include?('watch?v=')
|
95
75
|
@link_arr << link
|
96
76
|
count += 1
|
97
|
-
end
|
98
77
|
end
|
99
78
|
|
100
79
|
else
|
101
80
|
|
102
81
|
path.links_with(:text => /#{word}/).each do |link|
|
103
82
|
|
104
|
-
break if count > @max_downloads
|
105
|
-
|
106
|
-
# next if link.text.include?('Play')
|
83
|
+
break if count > @max_downloads
|
107
84
|
|
108
|
-
if link.text.include?(word) && link.
|
85
|
+
if link.text.include?(word) && link.href.include?('watch?v=')
|
109
86
|
@link_arr << link
|
110
87
|
count += 1
|
111
88
|
end
|
@@ -118,7 +95,7 @@ module BulkYoutube
|
|
118
95
|
def send_files(links)
|
119
96
|
you_convert = 'http://www.youtubeinmp4.com/'
|
120
97
|
links.each_with_index do |link, i|
|
121
|
-
clicked_link = link.
|
98
|
+
clicked_link = link.href
|
122
99
|
form = mec_get(you_convert).forms[0]
|
123
100
|
next if link.nil?
|
124
101
|
puts "printing link #{i}"
|
@@ -0,0 +1,140 @@
|
|
1
|
+
require "bulk_youtube/version"
|
2
|
+
require "bulk_youtube/globals"
|
3
|
+
require 'mechanize'
|
4
|
+
require 'nokogiri'
|
5
|
+
require 'benchmark'
|
6
|
+
|
7
|
+
|
8
|
+
class BenchmarkTest
|
9
|
+
include Globals
|
10
|
+
|
11
|
+
attr_accessor :search_path, :save_destination, :max_retries
|
12
|
+
|
13
|
+
def initialize(search_path=nil, save_destination=nil, max_downloads=2, max_retries=10)
|
14
|
+
@search_path = search_path
|
15
|
+
@save_destination = save_destination
|
16
|
+
@max_downloads = max_downloads
|
17
|
+
@max_retries = max_retries
|
18
|
+
end
|
19
|
+
|
20
|
+
|
21
|
+
# def you_convert
|
22
|
+
# retry_count = 0
|
23
|
+
# begin
|
24
|
+
# send_files(@link_arr)
|
25
|
+
# rescue Net::HTTP::Persistent::Error => e
|
26
|
+
# raise unless e.message =~ /too man connection resets/
|
27
|
+
# if retry_count > @max_retries
|
28
|
+
# puts "**** WARN: Mechanize retried connection reset #{@max_retries} times"
|
29
|
+
# raise
|
30
|
+
# end
|
31
|
+
# retry_count +=1
|
32
|
+
# Mechanize::HTTP::Agent.http.shutdown
|
33
|
+
# retry
|
34
|
+
# end
|
35
|
+
# end
|
36
|
+
|
37
|
+
def you_convert
|
38
|
+
begin
|
39
|
+
send_files(@link_arr)
|
40
|
+
rescue StandardError
|
41
|
+
false
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def grab_links(word=nil)
|
46
|
+
gather(word)
|
47
|
+
end
|
48
|
+
|
49
|
+
def show
|
50
|
+
arr = []
|
51
|
+
@link_arr.each do |link|
|
52
|
+
REPLACE.each do |r|
|
53
|
+
link.text.gsub!(r[0], r[1])
|
54
|
+
end
|
55
|
+
arr << link
|
56
|
+
end
|
57
|
+
arr
|
58
|
+
end
|
59
|
+
|
60
|
+
def links
|
61
|
+
@link_arr
|
62
|
+
end
|
63
|
+
|
64
|
+
|
65
|
+
def mec_get(path)
|
66
|
+
Mechanize.new.get(path)
|
67
|
+
end
|
68
|
+
|
69
|
+
|
70
|
+
def noko(path)
|
71
|
+
Nokogiri::HTML(path)
|
72
|
+
end
|
73
|
+
|
74
|
+
# def mec_get(path)
|
75
|
+
# time = Benchmark.measure do
|
76
|
+
# Mechanize.new.get(path)
|
77
|
+
# end
|
78
|
+
# puts time
|
79
|
+
# end
|
80
|
+
|
81
|
+
# returns itself
|
82
|
+
def gather(word)
|
83
|
+
@link_arr = []
|
84
|
+
path = mec_get(@search_path)
|
85
|
+
count = 0
|
86
|
+
|
87
|
+
if word.nil?
|
88
|
+
|
89
|
+
time = Benchmark.realtime do
|
90
|
+
|
91
|
+
###############################################
|
92
|
+
path.links_with(:href => /watch\?v=/).each do |youtube|
|
93
|
+
break if count > @max_downloads
|
94
|
+
|
95
|
+
puts youtube.href
|
96
|
+
|
97
|
+
include_time = Benchmark.realtime do
|
98
|
+
|
99
|
+
@link_arr << youtube if youtube
|
100
|
+
count += 1
|
101
|
+
|
102
|
+
end
|
103
|
+
puts "time of include check loop: #{include_time*1000} milliseconds"
|
104
|
+
end
|
105
|
+
self
|
106
|
+
end
|
107
|
+
puts "time of loop: #{time*1000} milliseconds"
|
108
|
+
|
109
|
+
else
|
110
|
+
|
111
|
+
path.links_with(:text => /#{word}/).each do |link|
|
112
|
+
|
113
|
+
break if count > @max_downloads
|
114
|
+
|
115
|
+
# next if link.text.include?('Play')
|
116
|
+
|
117
|
+
if link.text.include?(word) && link.href.include?('watch?v=')
|
118
|
+
@link_arr << link
|
119
|
+
count += 1
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
self
|
124
|
+
end
|
125
|
+
|
126
|
+
# requires an array
|
127
|
+
def send_files(links)
|
128
|
+
you_convert = 'http://www.youtubeinmp4.com/'
|
129
|
+
links.each_with_index do |link, i|
|
130
|
+
clicked_link = link.click.uri.to_s
|
131
|
+
form = mec_get(you_convert).forms[0]
|
132
|
+
next if link.nil?
|
133
|
+
puts "printing link #{i}"
|
134
|
+
form.field_with(:class => 'c3').value = clicked_link
|
135
|
+
path = "#{@save_destination}/#{i}.mp4"
|
136
|
+
form.submit.links[1].click.save_as(path)
|
137
|
+
puts "link #{1} successful"
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
data/lib/bulk_youtube/version.rb
CHANGED
@@ -0,0 +1,17 @@
|
|
1
|
+
require_relative '../lib/bulk_youtube/benchmark_testing'
|
2
|
+
|
3
|
+
describe "Benchmark" do
|
4
|
+
|
5
|
+
before do
|
6
|
+
path = 'https://www.youtube.com/watch?v=z5pJvhd3lFQ'
|
7
|
+
save = '/Users/mzakany/Desktop/folder'
|
8
|
+
@found = BenchmarkTest.new(path,save, 10)
|
9
|
+
end
|
10
|
+
|
11
|
+
it 'should' do
|
12
|
+
@found.grab_links.show
|
13
|
+
|
14
|
+
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
@@ -25,10 +25,17 @@ describe 'Full Page Playlist' do
|
|
25
25
|
end
|
26
26
|
|
27
27
|
it 'should grab this page' do
|
28
|
+
# path = 'https://www.youtube.com/watch?v=z5pJvhd3lFQ'
|
29
|
+
# save = '/Users/mzakany/Desktop/folder'
|
30
|
+
# found = BulkYoutube::Scrape.new(path,save, 5)
|
31
|
+
# found.grab_links.you_convert
|
32
|
+
end
|
33
|
+
|
34
|
+
it 'benchmark' do
|
28
35
|
path = 'https://www.youtube.com/watch?v=z5pJvhd3lFQ'
|
29
36
|
save = '/Users/mzakany/Desktop/folder'
|
30
|
-
found = BulkYoutube::Scrape.new(path,save)
|
31
|
-
|
37
|
+
found = BulkYoutube::Scrape.new(path,save, 5)
|
38
|
+
found.mec_get(path)
|
32
39
|
end
|
33
40
|
end
|
34
41
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: bulk_youtube
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.1
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2014-10-
|
12
|
+
date: 2014-10-16 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: nokogiri
|
@@ -98,6 +98,8 @@ executables: []
|
|
98
98
|
extensions: []
|
99
99
|
extra_rdoc_files: []
|
100
100
|
files:
|
101
|
+
- .floo
|
102
|
+
- .flooignore
|
101
103
|
- .gitignore
|
102
104
|
- .rspec
|
103
105
|
- Gemfile
|
@@ -106,8 +108,10 @@ files:
|
|
106
108
|
- Rakefile
|
107
109
|
- bulk_youtube.gemspec
|
108
110
|
- lib/bulk_youtube.rb
|
111
|
+
- lib/bulk_youtube/benchmark_testing.rb
|
109
112
|
- lib/bulk_youtube/globals.rb
|
110
113
|
- lib/bulk_youtube/version.rb
|
114
|
+
- spec/benchmark_spec.rb
|
111
115
|
- spec/embedded_playlists_spec.rb
|
112
116
|
- spec/full_page_playlist_spec.rb
|
113
117
|
- spec/spec_helper.rb
|
@@ -137,6 +141,7 @@ signing_key:
|
|
137
141
|
specification_version: 3
|
138
142
|
summary: Download Batches of youtube videos
|
139
143
|
test_files:
|
144
|
+
- spec/benchmark_spec.rb
|
140
145
|
- spec/embedded_playlists_spec.rb
|
141
146
|
- spec/full_page_playlist_spec.rb
|
142
147
|
- spec/spec_helper.rb
|