video_grabber 1.5.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3a484f6d9d5b763ae659792576bb8cc1d4e3a9ca
+  data.tar.gz: 8e7c4c25c9cfb9f06baf50f14f1e4e1c7ea0b37f
+SHA512:
+  metadata.gz: 0c92fa9ac5e5c9de371d2753cd70ae284f4daec88f038ac5df3f6d1a74262b73248c8166e710e7cc0c1c49cdab2a86ddde8b1a44204b96c004ff421b2f49f142
+  data.tar.gz: dad4f99ccd6fd9a64672645c84e6ed013daa00f300186d668f39ee9d3f4175005cc1df238ff77e88abd6ab47dce1a52e33b9c93fa20969aaf6357b1b25902a86

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.1.2
+before_install: gem install bundler -v 1.11.2

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in video_grabber.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,66 @@
+PATH
+  remote: .
+  specs:
+    video_grabber (1.4.0)
+      headless
+      nokogiri
+      selenium-webdriver (~> 2.53.4)
+      watir
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    childprocess (0.6.2)
+      ffi (~> 1.0, >= 1.0.11)
+    coderay (1.1.1)
+    commonwatir (4.0.0)
+    diff-lcs (1.2.5)
+    ffi (1.9.18)
+    headless (2.3.1)
+    method_source (0.8.2)
+    mini_portile2 (2.1.0)
+    nokogiri (1.7.1)
+      mini_portile2 (~> 2.1.0)
+    pry (0.10.4)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    rake (10.5.0)
+    rspec (3.4.0)
+      rspec-core (~> 3.4.0)
+      rspec-expectations (~> 3.4.0)
+      rspec-mocks (~> 3.4.0)
+    rspec-core (3.4.4)
+      rspec-support (~> 3.4.0)
+    rspec-expectations (3.4.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.4.0)
+    rspec-mocks (3.4.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.4.0)
+    rspec-support (3.4.1)
+    rubyzip (1.2.1)
+    selenium-webdriver (2.53.4)
+      childprocess (~> 0.5)
+      rubyzip (~> 1.0)
+      websocket (~> 1.0)
+    slop (3.6.0)
+    watir (5.0.0)
+      commonwatir (~> 4)
+      watir-webdriver
+    watir-webdriver (0.9.9)
+      selenium-webdriver (>= 2.46.2)
+    websocket (1.2.4)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.11)
+  pry
+  rake (~> 10.0)
+  rspec (~> 3.0)
+  video_grabber!
+
+BUNDLED WITH
+   1.12.5

data/LICENSE.md

@@ -0,0 +1,20 @@
+Copyright (c) 2017 Bridge2Think AG
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,96 @@
+# Video Grabber
+
+__Video Grabber__ is a simple tool to get video tags from a given URL. It includes advanced techniques to desobfuscate advertisements, run through embedded iframes, run javascript, and other frivolities that could usually and notably prevent one to fetch the wanted video.
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'video_grabber'
+```
+
+And then execute:
+
+```shell
+$ bundle
+```
+Or install it yourself as:
+
+```ruby
+$ gem install video_grabber
+```
+And require it in your application:
+
+```ruby
+irb(main):001:0> require 'video_grabber'
+=> true
+```
+
+## Requirements
+
+`video_grabber` relies on Firefox and [Headless](https://github.com/leonid-shevtsov/headless) (unless you specifically disable it). To get Headless working you will need linux and `xvfb`.
+
+Install `xvfb` on Debian:
+
+```shell
+sudo apt-get install xvfb
+```
+
+Headless is used to run Firefox inside a headless display (in background).
+
+
+## Usage
+
+Start the Scraper
+
+```ruby
+video_grabber = VideoGrabber.new(url: 'https://en.wikipedia.org/wiki/Big_Buck_Bunny').call
+```
+Fetch your links and shut down your scraper:
+
+```ruby
+video_grabber.fetch_videos
+=> ["<video  id=\"mwe_player_1\" poster=\"//upload.wikimedia.org/wikipedia/com...
+```
+
+If you want to fetch a new time those data, you can manually restart the service using:
+
+```ruby
+video_grabber.start
+```
+
+Or you can directly pass the param `keep_browser_open` during initialization.
+
+## Parameters
+
+ - **url:** The url of the resource containing the video(s)
+ -  **timeout:**  *(default: 60)* The timeout for the scraper. Will trigger a `VideoGrabber::Timeout` if the delay is met.
+ - **keep_browser_open** *(default: false)* If activated, will keep the scraper's browser open as long as you do not stop it (using the `stop` public method.). 
+ - **headless_enabled** *(default: true)* If disabled, will open your Firefox browser to crawl your links.
+- **attributes** This option enables you to pass html attributes that will be passed to your crawled links elements.
+ - **firefox_extension_path** If passed, your Scraper instance will run using the given extension (`.xpi` file). Useful if you want to benefit from an Adblocker for instance
+
+## Versioning
+
+__Video Grabber__ follows [Semantic Versioning 2.0](http://semver.org/).
+
+## Contributing
+
+1. Fork it ( https://github.com/bridge2think/video_grabber/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+## Contact
+
+Any question ? Feel free to contact me at `ss(at)bridge2think.com` .
+Any issue ? Open a [ticket](https://github.com/bridge2think/video_grabber/issues) !
+
+## License
+
+Copyright (c) 2017 Bridge2Think AG
+
+Released under the MIT license. See [LICENSE.md](https://github.com/bridge2think/video_grabber/blob/master/LICENSE.md) for more details.

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/fixtures/iframe_embedded_video.html

@@ -0,0 +1 @@
+<html><body><iframe><video src="iframe.mp4"></video></iframe></body></html>

data/fixtures/normal.html

@@ -0,0 +1 @@
+<html><body><video src='normal.mp4'></video></body></html>

data/lib/video_grabber.rb

@@ -0,0 +1,26 @@
+require 'video_grabber/application'
+require 'video_grabber/config.rb'
+require 'video_grabber/scraper.rb'
+require 'video_grabber/exceptions.rb'
+require 'selenium-webdriver'
+require 'headless'
+require 'nokogiri'
+require 'watir'
+require 'cgi'
+require 'pry'
+
+module VideoGrabber
+
+  class << self
+
+    attr_reader :application
+
+    def new(opts)
+      @application = VideoGrabber::Application.new(opts)
+    end
+
+    def call
+      application.call
+    end
+  end
+end

data/lib/video_grabber/application.rb

@@ -0,0 +1,22 @@
+module VideoGrabber
+  class Application
+
+    def initialize(options)
+      set_options(options)
+    end
+
+    def call
+      scraper = Scraper.new(config).start
+    end
+
+    private
+
+    def config
+      @config ||= ::VideoGrabber.config.dup
+    end
+
+    def set_options(options)
+      options.each { |k, v| config.send("#{k}=", v) }
+    end
+  end
+end

data/lib/video_grabber/config.rb

@@ -0,0 +1,34 @@
+module VideoGrabber
+
+  # Access point for the gem configurations.
+  #
+  # @return [VideoGrabber::Configuration] a configuration instance.
+  def self.config
+    @config ||= Configuration.new
+  end
+
+  # Configure hook used in the gem initializer. Convinient way to set all the
+  # gem configurations.
+  #
+  # example:
+  #   VideoGrabber.configure do |config|
+  #     config.timeout = 60
+  #   end
+  #
+  # @return [void]
+  def self.configure
+    yield config if block_given?
+  end
+
+  class Configuration
+
+    attr_accessor :url, :keep_browser_open, :timeout, :headless_enabled,
+                  :firefox_extension_path, :attributes
+
+    def initialize
+      @keep_browser_open = false
+      @timeout           = 60
+      @headless_enabled  = true
+    end
+  end
+end

data/lib/video_grabber/exceptions.rb

@@ -0,0 +1,6 @@
+module VideoGrabber
+  class Exception < StandardError; end
+  class BrowserIsClosed < Exception; end
+  class ExtensionError < Exception; end
+  class TimeOut < Exception; end
+end

data/lib/video_grabber/scraper.rb

@@ -0,0 +1,91 @@
+module VideoGrabber
+  class Scraper
+
+    attr_reader :url, :browser, :timeout, :keep_browser_open, :headless_enabled,
+                :firefox_extension_path, :profile, :attributes
+
+    def initialize(config)
+      @keep_browser_open      = config.keep_browser_open
+      @url                    = config.url
+      @timeout                = config.timeout
+      @headless_enabled       = config.headless_enabled
+      @firefox_extension_path = config.firefox_extension_path
+      @attributes             = config.attributes
+    end
+
+    def start
+      open_browser
+      browser.goto(url) ; self
+    rescue ::Net::ReadTimeout
+      stop
+      raise ::VideoGrabber::TimeOut
+    end
+
+    def stop
+      browser.close
+    end
+
+    def fetch_videos
+      links_list = []
+      links_list += browser.videos.map(&:html)
+
+      links_list += ::Nokogiri::HTML(browser.html).xpath('//iframe').map do |iframe_node|
+        ::Nokogiri::HTML(::CGI.unescapeHTML(iframe_node.to_s)).xpath('.//video').map{ |element| element.to_s }
+      end.flatten
+
+      links_list += begin
+        html = ::CGI.unescapeHTML(browser.html)
+        html = html.split('<video').map{|e| '<video ' + e if e.match('</video>')}.compact
+        html = html.map{|e| e.split('</video>')[0..-2].join('</video>') + '</video>' }
+      end
+
+      stop unless keep_browser_open
+
+      links_list = links_list.map{|element| element.split.join(" ") }.reject(&:empty?).uniq
+
+      add_attributes(links_list) || links_list
+    rescue ::Watir::Exception::Error
+      raise ::VideoGrabber::BrowserIsClosed, 'Please restart the scraper (scraper_instance.start), or keep the browser open'
+    end
+
+  private
+
+    def open_browser
+      start_headless
+
+      @profile       = ::Selenium::WebDriver::Firefox::Profile.new ; load_extension
+      client         = Selenium::WebDriver::Remote::Http::Default.new
+      client.timeout = timeout
+      @browser       = ::Watir::Browser.new(:firefox, profile: profile, http_client: client)
+    end
+
+    def start_headless
+      return unless headless_enabled
+
+      ::Headless.new.start
+    end
+
+    def load_extension
+      return unless firefox_extension_path
+
+      @profile.add_extension(firefox_extension_path)
+    rescue Selenium::WebDriver::Error::WebDriverError => e
+      raise ::VideoGrabber::ExtensionError, e
+    end
+
+    def add_attributes(list)
+      return unless attributes
+
+      list.map do |element|
+
+        parsed_element = Nokogiri::XML(element)
+
+        attributes.each do |key, value|
+          parsed_element.xpath('//video').first.set_attribute(key, value)
+        end
+
+        parsed_element.xpath('//video').to_s
+      end
+    end
+  end
+end

data/lib/video_grabber/version.rb

@@ -0,0 +1,3 @@
+module VideoGrabber
+  VERSION = "1.5.0"
+end

data/pkg_checksum

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require 'digest/sha2'
+
+gemname         = :video_grabber
+ARGV[0]         = File.read('VERSION.semver').chomp if ARGV[0].nil?
+built_gem_path  = "pkg/#{gemname}-#{ARGV[0]}.gem"
+checksum        = Digest::SHA512.new.hexdigest(File.read(built_gem_path))
+checksum_path   = "checksum/#{gemname}-#{ARGV[0]}.gem.sha512"
+
+File.open(checksum_path, 'w') { |f| f.write("#{checksum}\n") }

data/video_grabber.gemspec

@@ -0,0 +1,37 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'video_grabber/version'
+
+Gem::Specification.new do |spec|
+  spec.name                  = 'video_grabber'
+  spec.version               = VideoGrabber::VERSION
+  spec.authors               = ['sidney']
+  spec.email                 = ['ss@bridge2think.com']
+  spec.summary               = 'VideoGrabber is a simple tool to get video tags from a given URL'
+  spec.description           = 'VideoGrabber crawl headlessly websites to extract their videos'
+  spec.homepage              = 'https://github.com/bridge2think/video_grabber'
+  spec.license               = 'MIT'
+  spec.required_ruby_version = '>= 1.9.3'
+
+  # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
+  # delete this section to allow pushing this gem to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata['allowed_push_host'] = 'https://rubygems.org'
+  else
+    raise 'RubyGems 2.0 or newer is required to protect against public gem pushes.'
+  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.executables   = ['video_grabber']
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'selenium-webdriver', '~> 2.53.4'
+  spec.add_dependency 'watir'
+  spec.add_dependency 'headless'
+  spec.add_dependency 'nokogiri'
+  spec.add_development_dependency 'bundler', '~> 1.11'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'pry'
+end

metadata

@@ -0,0 +1,179 @@
+--- !ruby/object:Gem::Specification
+name: video_grabber
+version: !ruby/object:Gem::Version
+  version: 1.5.0
+platform: ruby
+authors:
+- sidney
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-03-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: selenium-webdriver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.53.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.53.4
+- !ruby/object:Gem::Dependency
+  name: watir
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: headless
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: VideoGrabber crawl headlessly websites to extract their videos
+email:
+- ss@bridge2think.com
+executables:
+- video_grabber
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.md
+- README.md
+- Rakefile
+- bin/video_grabber
+- fixtures/iframe_embedded_video.html
+- fixtures/normal.html
+- lib/video_grabber.rb
+- lib/video_grabber/application.rb
+- lib/video_grabber/config.rb
+- lib/video_grabber/exceptions.rb
+- lib/video_grabber/scraper.rb
+- lib/video_grabber/version.rb
+- pkg/video_grabber-1.1.0.gem
+- pkg_checksum
+- video_grabber-1.0.0.gem
+- video_grabber.gemspec
+homepage: https://github.com/bridge2think/video_grabber
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: VideoGrabber is a simple tool to get video tags from a given URL
+test_files: []
+has_rdoc: