hktv 1.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 480088f574c5b62191b13bf05b158af33316b6a0
+  data.tar.gz: b8cde4170b9e46f412663028818720cea5a21c51
+SHA512:
+  metadata.gz: db2ea07db61ffa58cdd23dd7043072bcf02d41f5c03c943c03a33c5cce3b34bfb6e8424118d893c48b881b290af38b16316fded9d69ecf0e3f065defba45f006
+  data.tar.gz: ded7e980b52a9555b8320c573dd02c7231be9d0360e66fe462f967c0f401d25bd0eb9eaa087b48b534016ad2624310a832e5a16624f52f485b4859026b8edc06

data/.gitignore

@@ -0,0 +1,15 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+*.gem

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in hktv.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Francis Chong
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,64 @@
+# HKTV
+
+Command line utilities to find and download HKTV videos.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'hktv', github: "siuying/ruby-hktv"
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it via:
+    
+    $ gem install hktv
+
+
+## Prerequisite
+
+- Ruby 2.x
+- Only tested on OS X Yosemite, it may work on Linux.
+- Requries ``ffmpeg`` for combine video.
+
+## Usage
+
+### Login to HKTV
+
+Login to HKTV. This is required for download video.
+
+    $ hktv login
+
+### List all video of HKTV
+
+Print a comma sepeated list of programs.
+
+    $ hktv list "選戰"
+    選戰 第1集 第1節
+    選戰 第1集 第2節
+    ...
+    選戰 第7集 第3節
+    選戰 第7集 第4節
+
+### Download an Episode
+
+Download all video files of an episode, and merge them into single file.
+
+    $ hktv download "選戰 第7集"
+    Downloading: 選戰 第7集 第1節
+    Downloading: 選戰 第7集 第2節
+    Downloading: 選戰 第7集 第3節
+    Downloading: 選戰 第7集 第4節
+    Merge videos into 選戰_第7集.mp4
+
+## Contributing
+
+1. Fork it ( https://github.com/siuying/ruby-hktv/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/bin/hktv

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require_relative '../lib/hktv/command'
+
+HKTV::Command.new.run

data/hktv.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'hktv/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "hktv"
+  spec.version       = HKTV::VERSION
+  spec.authors       = ["Francis Chong"]
+  spec.email         = ["francis@ignition.hk"]
+  spec.summary       = %q{Find and download HKTV videos.}
+  spec.description   = %q{Command line utilities to find and download HKTV videos.}
+  spec.homepage      = "https://github.com/siuying/ruby-hktv"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "pry", '~> 0'
+  spec.add_development_dependency "rspec", '~> 0'
+  spec.add_development_dependency "vcr", '~> 0'
+  spec.add_development_dependency "webmock", '~> 0'
+
+  spec.add_dependency "httparty", "~> 0.13", '>= 0.13.1'
+  spec.add_dependency "retriable", "~> 1.4", '>= 1.4.1'
+  spec.add_dependency 'commander', "~> 4.2", '>= 4.2.1'
+end

data/lib/hktv.rb

@@ -0,0 +1,241 @@
+require_relative "./hktv/version"
+
+require "httparty"
+require 'retriable'
+
+require "digest/md5"
+require "json"
+require 'securerandom'
+
+class HKTV
+  include HTTParty
+
+  # set by auth
+  attr_accessor :access_token, :expires_date, :refresh_token
+
+  # set by token
+  attr_accessor :user_id, :user_level, :ott_token, :ott_expires_date
+
+  base_uri 'webservices.hktv.com.hk'
+  headers "Content_type" => "application/x-www-form-urlencoded", "Accept" => "*/*"
+
+  API_BASE = "/"
+  API_TOKEN = "account/token"
+  API_FEATURE = "lists/getFeature"
+  API_PLAYLIST = "playlist/request"  
+
+  API_SECRET = "43e814b31f8764756672c1cd1217d775"
+  API_KI = "12"
+  API_VID = "1"
+
+  API_MUID = "0"
+  API_DEVICE = "USB Android TV"
+  API_MANUF = "hktv-ruby"
+  API_MODEL = "Ruby"
+  API_OS = "0.1.0"
+  API_MX_RES = "1920"
+  API_NETWORK = "fixed"
+
+  # hardcoded user in HKTV app
+  API_USERNAME = "hktv_ios"
+  API_PASSWORD = "H*aK#)HM248"
+
+  def initialize(uuid: SecureRandom.uuid, access_token: nil, expires_date: nil, refresh_token: nil, user_id: "1", user_level: nil, ott_token: nil, ott_expires_date: nil)
+    @uuid = uuid
+    @access_token = access_token
+    @expires_date = expires_date
+    @refresh_token = refresh_token
+    @ott_token = ott_token    
+    @ott_expires_date = ott_expires_date
+    @user_id = user_id
+    @user_level = user_level
+  end
+
+  # return true if client has authenticated, false otherwise
+  def authenticated?
+    if @expires_date && Time.now > @expires_date
+      @access_token = nil
+      @expires_date = nil
+      @refresh_token = nil
+    end
+
+    !@access_token.nil?
+  end
+
+  # return true if client needs ott token
+  def needs_ott_token?
+    return @ott_token.nil? || (@ott_expires_date && Time.now > @ott_expires_date)
+  end
+
+  # Authenticate user with given username/password
+  # return true if success
+  def auth(username, password)
+    options = {
+      "grant_type" => "password", 
+      "username" => username,
+      "password" => password
+    }
+
+    auth = post_json("https://www.hktvmall.com:443/hktvwebservices/oauth/token?rand=#{Time.now.to_i}", body: options, basic_auth: {username: API_USERNAME, password: API_PASSWORD})
+    @access_token = auth["access_token"]
+    @expires_date = Time.now + auth["expires_in"]
+    @refresh_token = auth["refresh_token"]
+    !@access_token.nil?
+  end
+
+  # get the OTT token
+  def ott_token
+    result = nil
+
+    # hktv api just fail for unknown reason, retry to workaround it
+    Retriable.retriable(tries: 20, base_interval: 1.0) do
+      if authenticated?
+        result = get_json("https://www.hktvmall.com:443/hktvwebservices/v1/hktv/ott/token?rand=#{Time.now.to_i}", headers: headers)
+      else
+        ts = Time.now.to_i
+        options = {
+          "ki" => API_KI, 
+          "ts" => ts.to_s,
+          "s" => sign_request(API_TOKEN, ts, [API_KI, API_MUID]),
+          "muid" => API_MUID
+        }
+        result = self.class.post(API_BASE + API_TOKEN, body: options, headers: headers)
+      end
+
+      if result["errors"]
+        raise result["errors"].first["message"]
+      end
+
+      result
+    end
+
+    @user_id = result["user_id"]
+    @user_level = result["user_level"]
+    @ott_token = result["token"]
+
+    !@user_id.nil?
+  end
+
+  def logout
+    @user_id = nil
+    @user_level = nil
+    @ott_token = nil
+
+    @access_token = nil
+    @expires_in = nil
+    @refresh_token = nil
+
+    if authenticated?
+      self.class.post("https://www.hktvmall.com:443/hktvwebservices/v1/customers/current/logout", headers: headers)["success"]
+    else
+      true
+    end
+  end
+
+  # get a playlist of given video
+  # @param video_id the video ID
+  # @return URL to the playlist
+  def playlist(video_id="1")
+    self.ott_token if needs_ott_token?
+
+    ts = Time.now.to_i
+    signature = sign_request(API_PLAYLIST, ts, [API_DEVICE, API_KI, API_MODEL, API_MANUF, API_MX_RES, API_NETWORK, API_OS, @ott_token, @uuid, @user_id, video_id])
+    options = {
+      "d" => API_DEVICE,
+      "ki" => API_KI,
+      "mdl" => API_MODEL,
+      "mf" => API_MANUF,
+      "mxres" => API_MX_RES,
+      "net" => API_NETWORK,
+      "os" => API_OS,
+      "t" => @ott_token,
+      "udid" => @uuid,
+      "uid" => @user_id,
+      "vid" => video_id,
+      "ts" => ts.to_s,
+      "s" => signature
+    }
+    self.class.post(API_BASE + API_PLAYLIST, body: options)["m3u8"]
+  end
+
+  # list featured video on HKTV
+  # @return nested array of video
+  def features(lang="zh-Hant", count=999)
+    self.ott_token if needs_ott_token?
+
+    ts = Time.now.to_i
+    options = {
+      "lang" => lang,
+      "lim" => count.to_s,
+      "lut" => "0",
+      "_" => ts.to_s,
+      "ofs" => "0"
+    }
+    self.class.get("http://ott-www.hktvmall.com/api/lists/getFeature", query: options)["videos"]
+  end
+
+  # list programs on HKTV
+  # @return nested array of video
+  def programs(lang="zh-Hant", count=999)
+    ts = Time.now.to_i
+    options = {
+      "lang" => lang,
+      "lim" => count.to_s,
+      "lut" => "0",
+      "_" => ts.to_s,
+      "ofs" => "0"
+    }
+    self.class.get("http://ott-www.hktvmall.com/api/lists/getProgram", query: options)["videos"]
+  end
+
+  def to_hash
+    json = {
+      uuid: @uuid,
+      access_token: @access_token,
+      refresh_token: @refresh_token,
+      user_id: @user_id,
+      user_level: @user_level,
+      ott_token: @ott_token
+    }
+    json[:expires_date] = @expires_date.to_i if @expires_date
+    json[:ott_expires_date] = @ott_expires_date.to_i if @ott_expires_date
+    return json
+  end
+
+  def self.from_hash(json)
+    if json[:"expires_date"]
+      json[:expires_date] = Time.at(json[:expires_date])
+    end
+    if json[:ott_expires_date]
+      json[:ott_expires_date] = Time.at(json[:ott_expires_date])
+    end
+    return HKTV.new(json)
+  end
+
+  private
+  def headers
+    if authenticated?
+      return {
+        "Authorization" => "Bearer #{@access_token}"
+      }
+    else
+      return {}
+    end
+  end
+
+  def post_json(path, options={})
+    response = self.class.post(path, options)
+    data = response.body
+    JSON.parse(data)
+  end
+
+  def get_json(path, options={})
+    response = self.class.get(path, options)
+    data = response.body
+    JSON.parse(data)
+  end
+
+  def sign_request(path, timestamp, params=[])
+    return Digest::MD5.hexdigest(path + params.join("") + API_SECRET + timestamp.to_s)
+  end
+end

data/lib/hktv/command.rb

@@ -0,0 +1,183 @@
+require 'commander'
+require 'fileutils'
+require 'csv'
+require 'open3'
+require_relative './version'
+require_relative '../hktv'
+
+class HKTV
+  class Command
+    include Commander::Methods
+
+    CONFIG_DIR = "#{Dir.home}/.hktv"
+    CONFIG_FILE = "#{Dir.home}/.hktv/hktv.json"
+
+    def load
+      hktv = nil
+
+      if !File.exists?(CONFIG_DIR)
+        FileUtils.mkdir_p(CONFIG_DIR)
+      end
+
+      if File.exists?(CONFIG_FILE)
+        hktv = HKTV.from_hash(JSON(File.open(CONFIG_FILE).read, symbolize_names: true))
+      else
+        hktv = HKTV.new
+      end
+
+      return hktv
+    end
+
+    def save(hktv)
+      if !File.exists?(CONFIG_DIR)
+        FileUtils.mkdir_p(CONFIG_DIR)
+      end
+
+      File.open(CONFIG_FILE, 'w') do |f|
+        f.write(JSON.generate(hktv.to_hash))
+      end
+    end
+
+    def run
+      program :name, 'hktv'
+      program :version, HKTV::VERSION
+      program :description, 'Lookup HKTV videos'
+      program :help_formatter, :compact
+
+      command :login do |c|
+        c.syntax = 'hktv login'
+        c.description = 'Login to HKTV'
+        c.action do |args, options|
+          username = ask("username: ")
+          password = ask("password: ") { |q| q.echo = "*" }
+
+          hktv = self.load
+          hktv.logout if hktv.authenticated?
+
+          if hktv.auth(username, password)
+            puts "Logged in"
+            self.save(hktv)
+          end
+        end
+      end
+
+      command :list do |c|
+        c.syntax = 'hktv list [episode-title]'
+        c.description = 'Print a comma sepeated list of programs of HKTV'
+        c.option '--keys keys', String, 'Output data keys, by default "title", available: [title, video_id, category, thumbnail, url, duration]'
+        c.option '--playlist', 'Fetch the playlist URL. By default the URL is not fetched.'
+
+        c.action do |args, options|
+          options.default category: "DRAMA", playlist: false, keys: "title"
+          hktv = self.load
+          programs = extract_root_videos(hktv.programs)
+          keys = options.keys.split(",")
+          title = args[0]
+
+          if options.category
+            programs = programs.select {|program| program["category"] == options.category }
+          end
+
+          if title
+            programs = programs.select {|program| program["title"].include?(title) }
+          end
+
+          if options.playlist
+            programs = programs.select {|program| program["url"] = hktv.playlist(program["video_id"]) }
+          end
+
+          if options.playlist && !keys.include?("url") && options.keys == "title,video_id"
+            keys << "url"
+          end
+
+          rows = programs.map do |program|
+            keys.collect do |key|
+              program[key]
+            end.to_csv
+          end
+
+          puts rows.join("")
+        end
+      end
+
+      command :download do |c|
+        c.syntax = 'hktv download [episode-title] (output-filename)'
+        c.description = 'Download an episode of HKTV program.'
+
+        c.action do |args, options|
+          hktv = self.load
+
+          unless hktv.authenticated?
+            puts "You have not login! Try \"hktv login\"" 
+            raise "Not logged in."
+          end
+
+          title = args[0]
+          filename = args[1]
+          raise "Missing episode title" if title.nil?
+
+          filename = filename_with_title(title, ".mp4") if filename.nil?
+          programs = extract_root_videos(hktv.programs)
+          programs = programs.select {|program| program["title"].include?(title) }
+
+          if programs.size == 0
+            puts "No video matching \"#{title}\""
+            raise "Video not found"
+          end
+
+          # fetch playlist url
+          programs.each {|program| program["url"] = hktv.playlist(program["video_id"]) }
+
+          # download playlist and merge them
+          download_and_merge_programs(programs, filename)      
+        end
+      end
+
+      run!
+    end
+
+    private
+    def extract_root_videos(videos)
+      videos.collect do |video|
+        if video["child_nodes"]
+          extract_root_videos(video["child_nodes"])
+        else
+          video
+        end
+      end.flatten
+    end
+
+    def filename_with_title(title, ext=".ts")
+      title.gsub(/\s/, "_").gsub(/\//, "") + ext
+    end
+
+    def download_and_merge_programs(videos, output)
+      temp_files = videos.map {|video| filename_with_title(video["title"]) }
+      failed = false
+
+      begin
+        # download the video and convert them into ts file
+        # https://trac.ffmpeg.org/wiki/Concatenate
+        videos.each do |video|
+          url = video["url"]
+          title = video["title"]
+          puts "Downloading: #{title}"
+          `ffmpeg -i \"#{url}\" -c copy -bsf:v h264_mp4toannexb -f mpegts \"#{filename_with_title(title)}\" 2> /dev/null`
+          if $?.to_i != 0
+            puts "Failed download file."
+          end
+        end
+
+        # losslessly merge these ts
+        puts "Merge videos into #{output}"
+        `ffmpeg -f mpegts -i \"concat:#{temp_files.join("|")}\" -c copy -bsf:a aac_adtstoasc \"#{output}\" 2> /dev/null`
+        if $?.to_i != 0
+          puts "Failed merging file."
+        end
+      ensure 
+        puts "Remove temp file: #{temp_files.join(" ")}"
+        `rm \"#{temp_files.join("\" \"")}\"`
+      end
+    end
+  end
+end