copycasts 0.0.2 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/LICENSE.txt +1 -1
- data/README.md +20 -10
- data/VERSION +1 -1
- data/copycasts.gemspec +1 -0
- data/lib/copycasts.rb +30 -11
- metadata +15 -2
- data/lib/markdown +0 -122
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7b869318d2c2200d35db29241d3e080d37c4dd71
|
4
|
+
data.tar.gz: ef78807c096372e7e88e7404444f2482f6e1cdf8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6bda003314c3814ef3c84fbe0b4491a450e85604e5c492b28241a68e02ad3082fd449827680653839e0020c2ae8275fc1586f749ee4d1ca5c239345c4420749d
|
7
|
+
data.tar.gz: 3e5a2663725679a812ddb197972c714e7f34fe5a0b29cc3cdcf5904388bb64c0049f617fcf794d7611fde4cc1c95f2adc8c4ba5f3d28fdc060e9e691dbe46256
|
data/LICENSE.txt
CHANGED
data/README.md
CHANGED
@@ -1,18 +1,10 @@
|
|
1
1
|
# Copycasts
|
2
2
|
|
3
|
-
|
3
|
+
You are able to access railscast video offline now by downloading them. This gem will crawl railcast videos and download to your local storage.
|
4
4
|
|
5
5
|
## Installation
|
6
6
|
|
7
|
-
|
8
|
-
|
9
|
-
gem 'copycasts'
|
10
|
-
|
11
|
-
And then execute:
|
12
|
-
|
13
|
-
$ bundle
|
14
|
-
|
15
|
-
Or install it yourself as:
|
7
|
+
Install it the gem globally as:
|
16
8
|
|
17
9
|
$ gem install copycasts
|
18
10
|
|
@@ -29,3 +21,21 @@ Run using this command in your terminal:
|
|
29
21
|
3. Commit your changes (`git commit -am 'Add some feature'`)
|
30
22
|
4. Push to the branch (`git push origin my-new-feature`)
|
31
23
|
5. Create new Pull Request
|
24
|
+
|
25
|
+
##Coming soon
|
26
|
+
|
27
|
+
1. Upload to aws storage
|
28
|
+
2. Upload to dropbox
|
29
|
+
3. Options to download different file type [zip, mp4, m4v, webm, ogv]
|
30
|
+
4. Using resque / multithreading
|
31
|
+
5. Any others ??
|
32
|
+
|
33
|
+
##Thanks
|
34
|
+
|
35
|
+
Thanks to railcast providing free videos for our learning purpose.
|
36
|
+
Reference: [RailsCasts, Ryan Bates](http://railscasts.com/)
|
37
|
+
|
38
|
+
## Anything I missed?
|
39
|
+
|
40
|
+
If there's anything at all that you need or want to know, please email either
|
41
|
+
[Nicholas Ng](mailto:secret@live.com.my) and I ll get back to you soon. :D
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.0.
|
1
|
+
0.0.3
|
data/copycasts.gemspec
CHANGED
@@ -21,4 +21,5 @@ Gem::Specification.new do |spec|
|
|
21
21
|
spec.add_development_dependency "rake"
|
22
22
|
spec.add_development_dependency "nokogiri", "~> 1.5", ">= 1.5.6"
|
23
23
|
spec.add_development_dependency "open-uri", "~> 1.5", ">= 1.5.6"
|
24
|
+
spec.add_development_dependency "progressbar", "~> 0.21.0"
|
24
25
|
end
|
data/lib/copycasts.rb
CHANGED
@@ -1,12 +1,12 @@
|
|
1
1
|
require 'nokogiri'
|
2
2
|
require 'open-uri'
|
3
3
|
require 'net/http'
|
4
|
+
require 'progressbar'
|
4
5
|
|
5
6
|
module Copycasts
|
6
7
|
|
7
8
|
class Crawling
|
8
9
|
TARGET_URL = 'http://railscasts.com'
|
9
|
-
attr_accessor :page
|
10
10
|
|
11
11
|
def initialize(options = {})
|
12
12
|
@pages = options[:page] || maximum_page
|
@@ -14,16 +14,18 @@ module Copycasts
|
|
14
14
|
|
15
15
|
def get_links
|
16
16
|
casts_list = []
|
17
|
-
puts "Start crawling
|
18
|
-
|
19
|
-
|
17
|
+
puts "Start crawling page "
|
18
|
+
|
19
|
+
for index in 1..2
|
20
|
+
print "#{index}"
|
21
|
+
print ", " if index != @pages
|
20
22
|
target_page = Nokogiri::HTML(open(TARGET_URL + "/?type=free&page=#{index}"))
|
21
23
|
target_page.css('.watch a:first').each do |link|
|
22
24
|
link_without_autoplay = link['href'].to_s.sub('?autoplay=true','')
|
23
25
|
casts_list << link_without_autoplay
|
24
26
|
end
|
25
27
|
end
|
26
|
-
puts "
|
28
|
+
puts "\n"
|
27
29
|
casts_list
|
28
30
|
end
|
29
31
|
|
@@ -41,29 +43,46 @@ module Copycasts
|
|
41
43
|
end
|
42
44
|
|
43
45
|
def mp4_video_links
|
46
|
+
count = 0
|
44
47
|
mp4_links = []
|
45
|
-
|
48
|
+
page_links = get_links
|
49
|
+
|
50
|
+
puts "Start crawling for download target"
|
51
|
+
progress = ProgressBar.new("Crawling:", page_links.length)
|
52
|
+
|
53
|
+
page_links.each do |video_link|
|
46
54
|
video_page = Nokogiri::HTML(open(TARGET_URL + "/" + video_link))
|
47
55
|
link = video_page.css('.downloads li[3] a').first
|
48
56
|
mp4_links << link.values.first
|
57
|
+
count += 1
|
58
|
+
progress.set(count)
|
49
59
|
end
|
60
|
+
puts "\n"
|
50
61
|
mp4_links
|
51
62
|
end
|
52
63
|
|
53
64
|
def download_videos
|
65
|
+
downloaded = 0
|
54
66
|
mp4_video_links.each do |video_link|
|
67
|
+
count = 0
|
55
68
|
uri = URI.parse(video_link)
|
56
69
|
file_name = video_link.split("/").last
|
57
70
|
|
58
71
|
Net::HTTP.start(uri.host) do |http|
|
59
|
-
|
60
|
-
|
61
|
-
open(file_name, "wb") do |file|
|
62
|
-
|
72
|
+
response = http.request_head(uri.request_uri)
|
73
|
+
progress = ProgressBar.new("#{downloaded} downloaded", response['content-length'].to_i)
|
74
|
+
File.open(file_name, "wb") do |file|
|
75
|
+
http.get(uri.request_uri) do |request_return|
|
76
|
+
file.write(request_return)
|
77
|
+
count += request_return.length
|
78
|
+
progress.set(count)
|
79
|
+
end
|
63
80
|
end
|
64
81
|
end
|
65
|
-
|
82
|
+
downloaded += 1
|
66
83
|
end
|
84
|
+
|
85
|
+
puts "Downloaded all files successfully!"
|
67
86
|
end
|
68
87
|
end
|
69
88
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: copycasts
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nicholas
|
@@ -78,6 +78,20 @@ dependencies:
|
|
78
78
|
- - '>='
|
79
79
|
- !ruby/object:Gem::Version
|
80
80
|
version: 1.5.6
|
81
|
+
- !ruby/object:Gem::Dependency
|
82
|
+
name: progressbar
|
83
|
+
requirement: !ruby/object:Gem::Requirement
|
84
|
+
requirements:
|
85
|
+
- - ~>
|
86
|
+
- !ruby/object:Gem::Version
|
87
|
+
version: 0.21.0
|
88
|
+
type: :development
|
89
|
+
prerelease: false
|
90
|
+
version_requirements: !ruby/object:Gem::Requirement
|
91
|
+
requirements:
|
92
|
+
- - ~>
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: 0.21.0
|
81
95
|
description: Offline watch free episode of railscast videos for personal usage.
|
82
96
|
email:
|
83
97
|
- secret@live.com.my
|
@@ -95,7 +109,6 @@ files:
|
|
95
109
|
- bin/copycasts-start-crawl
|
96
110
|
- copycasts.gemspec
|
97
111
|
- lib/copycasts.rb
|
98
|
-
- lib/markdown
|
99
112
|
homepage: https://github.com/nichthinkof30/copycasts
|
100
113
|
licenses:
|
101
114
|
- MIT
|
data/lib/markdown
DELETED
@@ -1,122 +0,0 @@
|
|
1
|
-
gem uninstall copycasts
|
2
|
-
gem build copycasts.gemspec
|
3
|
-
gem install copycasts
|
4
|
-
|
5
|
-
require 'copycasts'
|
6
|
-
a = CopyCasts.new({page: 1})
|
7
|
-
a.download_videos
|
8
|
-
|
9
|
-
types = {}
|
10
|
-
file_links = []
|
11
|
-
|
12
|
-
types[:zip] = 2 if options[:zip]
|
13
|
-
types[:mp4] = 3 if options[:mp4]
|
14
|
-
types[:m4v] = 4 if options[:m4v]
|
15
|
-
types[:webm] = 5 if options[:webm]
|
16
|
-
types[:ogv] = 6 if options[:ogv]
|
17
|
-
|
18
|
-
types = { zip: 2, mp4: 3, m4v: 4, webm: 5, ogv: 6 } if options[:all]
|
19
|
-
|
20
|
-
types.each_pair do |key, value|
|
21
|
-
puts "Crawling "
|
22
|
-
get_links.each do |link|
|
23
|
-
file_link = Nokogiri::HTML(open(TARGET_URL + "/" + link))
|
24
|
-
url = url.css('.downloads li[#{value}] a').first
|
25
|
-
file_links << url.values.first
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
file_links
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
Gem::Specification.new do |s|
|
34
|
-
s.name = "copycasts"
|
35
|
-
s.version = "0.0.2"
|
36
|
-
s.authors = ["Nicholas Ng"]
|
37
|
-
s.email = ["secret@live.com.my"]
|
38
|
-
s.homepage = ""
|
39
|
-
s.summary = %q{Offline watch free episode of railscast videos for personal usage.}
|
40
|
-
s.description = %q{Download videos from video source}
|
41
|
-
|
42
|
-
s.add_development_dependency "nokogiri", "~> 1.5", ">= 1.5.6"
|
43
|
-
s.add_development_dependency "open-uri", "~> 1.5", ">= 1.5.6"
|
44
|
-
s.add_development_dependency "rake", "~> 10.1", ">= 10.1.0"
|
45
|
-
|
46
|
-
s.files = [
|
47
|
-
"lib/copycasts.rb"
|
48
|
-
]
|
49
|
-
s.require_paths = ["lib"]
|
50
|
-
end
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
require 'nokogiri'
|
57
|
-
require 'open-uri'
|
58
|
-
require 'net/http'
|
59
|
-
|
60
|
-
class CopyCasts
|
61
|
-
TARGET_URL = 'http://railscasts.com'
|
62
|
-
attr_accessor :page
|
63
|
-
|
64
|
-
def initialize(options = {})
|
65
|
-
@pages = options[:page] || maximum_page
|
66
|
-
|
67
|
-
end
|
68
|
-
|
69
|
-
def get_links
|
70
|
-
casts_list = []
|
71
|
-
puts "Crawling casts list..."
|
72
|
-
for index in 1..@pages
|
73
|
-
target_page = Nokogiri::HTML(open(TARGET_URL + "/?type=free&page=#{index}"))
|
74
|
-
target_page.css('.watch a:first').each do |link|
|
75
|
-
link_without_autoplay = link['href'].to_s.sub('?autoplay=true','')
|
76
|
-
casts_list << link_without_autoplay
|
77
|
-
end
|
78
|
-
end
|
79
|
-
puts "Finish crawling."
|
80
|
-
casts_list
|
81
|
-
end
|
82
|
-
|
83
|
-
def maximum_page
|
84
|
-
target_page = Nokogiri::HTML(open(TARGET_URL + "/?type=free"))
|
85
|
-
ret = 0
|
86
|
-
target_page.css('.pagination a').each do |a|
|
87
|
-
if !(a.content.match(/\A[+-]?\d+?(\.\d+)?\Z/) == nil) #not number
|
88
|
-
if a.content.to_i > 0
|
89
|
-
ret = a.content.to_i
|
90
|
-
end
|
91
|
-
end
|
92
|
-
end
|
93
|
-
ret
|
94
|
-
end
|
95
|
-
|
96
|
-
def file_links
|
97
|
-
mp4_links = []
|
98
|
-
get_links.each do |video_link|
|
99
|
-
video_page = Nokogiri::HTML(open(TARGET_URL + "/" + video_link))
|
100
|
-
link = video_page.css('.downloads li[3] a').first
|
101
|
-
mp4_links << link.values.first
|
102
|
-
end
|
103
|
-
mp4_links
|
104
|
-
end
|
105
|
-
|
106
|
-
def download_videos
|
107
|
-
mp4_video_links.each do |video_link|
|
108
|
-
uri = URI.parse(video_link)
|
109
|
-
file_name = video_link.split("/").last
|
110
|
-
|
111
|
-
Net::HTTP.start(uri.host) do |http|
|
112
|
-
puts "Start downloading #{file_name}..."
|
113
|
-
response = http.get(uri.request_uri)
|
114
|
-
open(file_name, "wb") do |file|
|
115
|
-
file.write(response.body)
|
116
|
-
end
|
117
|
-
end
|
118
|
-
puts "Downloaded successfully!"
|
119
|
-
end
|
120
|
-
end
|
121
|
-
|
122
|
-
end
|