stupeflixclient 0.0.1

data/lib/stupeflixclient/connection.rb

@@ -0,0 +1,123 @@
+require 'net/http'
+require 'uri'
+
+module Stupeflixclient
+
+  class Connection
+    def initialize(server, base_url)
+      @server = server
+      @base_url = base_url
+      @MAX_NETWORK_RETRY = 1
+      @debuglevel = 0
+    end
+
+    def request_get( resource, args = nil, headers={})
+      return request(resource, "get", args, body = nil, filename = nil, headers=headers)
+    end
+
+    def request_delete( resource, args = nil, headers={})
+      return request(resource, "delete", args, headers=headers)
+    end
+
+    def request_head( resource, args = nil, headers={})
+      return request(resource, "head", args, headers=headers)
+    end
+
+    def request_post( resource, args = nil, body = nil, filename=nil, headers={})
+      return request(resource, "post", args , body = body, filename=filename, headers=headers)
+    end
+
+    def request_put(resource, args = nil, body = nil, filename=nil, headers={}, sendcallback = nil)
+      dump("In request_put in connection", resource, "PUT")
+      resp = request(resource, "put", args , body = body, filename=filename, headers=headers, sendcallback = sendcallback)
+      return resp
+    end
+
+    def dump( message, request_uri, method)
+      if @debuglevel > 0
+        print message, " " , time.asctime, " " , request_uri, "\n"
+      end
+    end
+
+    def request_method(verb)
+      Net::HTTP.const_get(verb.to_s.capitalize)
+    end
+
+    def fetch(uri_str, limit = 10)
+      # You should choose better exception.
+      raise ArgumentError, 'HTTP redirect too deep' if limit == 0
+
+      response = Net::HTTP.get_response(URI.parse(uri_str))
+      case response
+      when Net::HTTPSuccess     then response
+      when Net::HTTPRedirection then fetch(response['location'], limit - 1)
+      else
+        response.error!
+      end
+
+    end
+
+    def request( resource, method = "get", args = nil, body = nil, filename=nil, headers={}, sendcallback = nil)
+      params = nil
+      path = resource
+
+      if headers == nil
+        headers = {}
+      end
+
+      headers['User-Agent'] = 'Basic Agent'
+
+      # TEMPORARY : add support for streaming
+      if method != "get"
+        if not body and filename
+          bodystream = File.open(filename, 'r')
+        end
+      end
+
+      if args
+        path += "?" + urllib.urlencode(args)
+      end
+
+      url = 'http://' + @server + @base_url + path
+
+      dump("Connection Request starting", path, method.upcase)
+
+      for i in 0..@MAX_NETWORK_RETRY
+        begin
+          #        http.set_debug_output $stdout
+          #        headers['Expect'] = '100-Continue'
+
+          response = nil
+          while true
+            url = URI.parse(url)
+            request = request_method(method).new(url.path + "?" + url.query, headers)
+            if bodystream
+              request.body_stream = bodystream
+            elsif body
+              request.body = body
+            end
+            http = Net::HTTP.new(url.host, url.port)
+            result = http.start {|http|
+              response = http.request(request)
+            }
+            case response
+            when Net::HTTPRedirection
+              then
+              url = response['location']
+            else
+              break
+            end
+          end
+          break
+        rescue StandardError => e
+          if i == (@MAX_NETWORK_RETRY - 1)
+            raise
+          end
+        end
+      end
+      response["status"] = response.code.to_s
+      return {'headers' => response, 'body' => response.body}
+    end
+
+  end
+end

data/lib/stupeflixclient/constants.rb

@@ -0,0 +1,4 @@
+module Stupeflixclient::Constants
+  SERVER_NAME = "http://services.stupeflix.com"
+  BASE_ENDPOINT = 'stupeflix-1.0'
+end

data/lib/stupeflixclient/settings.rb

@@ -0,0 +1,4 @@
+module Stupeflixclient::Settings
+  ACCESS_KEY = ENV['stupeflix_access_key']
+  ACCESS_SECRET = ENV['stupeflix_access_secret']
+end

data/lib/stupeflixclient/stupeflix_base.rb

@@ -0,0 +1,273 @@
+require 'openssl'
+require 'digest'
+require 'base64'
+require 'stupeflixclient/connection'
+require 'digest/md5'
+module Stupeflixclient
+  class StupeflixBase
+    def initialize( accessKey, privateKey, host = "http://services.stupeflix.com", service = 'stupeflix-1.0', debug = false)
+      @accessKey = accessKey
+      @privateKey = privateKey
+      len = host.length - 7
+      if host[host.length - 1, 1] == "/"
+        host = host[0, host.length - 1]
+      end
+
+      @host = host[7,len]
+      @base_url = service
+      @debug = debug
+      @service = service
+      @TEXT_XML_CONTENT_TYPE = "text/xml"
+      @APPLICATION_ZIP_CONTENT_TYPE = "application/zip"
+      @APPLICATION_JSON_CONTENT_TYPE = "application/json"
+      @APPLICATION_URLENCODED_CONTENT_TYPE = "application/x-www-form-urlencoded"
+      @PROFILES_PARAMETER = "Profiles"
+      @XML_PARAMETER = "ProfilesXML"
+      @MARKER_PARAMETER = "Marker"
+      @MAXKEYS_PARAMETER = "MaxKeys"
+      # Currently there is only the Marker parameter (used for partial enumeration)
+      @parametersToAdd = [@MARKER_PARAMETER, @MAXKEYS_PARAMETER]
+      @sleepTime = 1.0
+      @maxRetry = 4
+      @base = true
+    end
+
+    def connectionGet
+      return Connection.new(@host, "/" + @base_url)
+    end
+
+    def paramString( parameters)
+      paramStr = ""
+      if parameters != nil
+        for p in @parametersToAdd
+          if parameters.include?(p)
+            paramStr += sprintf( "%s\n%s\n", p, parameters[p])
+          end
+        end
+      end
+      return paramStr
+    end
+
+    def strToSign( method, resource, md5, mime, datestr, parameters)
+      paramStr = paramString(parameters)
+      stringToSign  = sprintf( "%s\n%s\n%s\n%s\n%s\n%s", method, md5, mime, datestr, '/' + @service + resource, paramStr)
+      return stringToSign
+    end
+
+    def sign( strToSign, secretKey)
+      digest  = OpenSSL::Digest::Digest.new('sha1')
+      return OpenSSL::HMAC.hexdigest(digest, secretKey, strToSign)
+    end
+
+    def signUrl( url, method, md5, mime, parameters = {})
+      now =Time.now.to_i
+      strToSign = strToSign(method, url, md5, mime, now, parameters)
+      signature = sign(strToSign, @privateKey)
+      url += sprintf( "?Date=%s&AccessKey=%s&Signature=%s", now, @accessKey,signature)
+      if parameters
+        parameters.each_pair do |k,v|
+          url += sprintf("&%s=%s", k,v)
+        end
+      end
+      return url
+    end
+
+    def md5FileOrBody( filename, body = nil)
+      md5 = Digest::MD5.new()
+
+      if body != nil
+        md5.update(body)
+      else
+        chunksize=1024
+        f = File.open(filename, 'r')
+
+        while true
+          chunk = f.read(chunksize)
+          if not chunk
+            break
+          end
+          md5.update(chunk)
+        end
+        f.close
+      end
+
+      digest = md5.digest
+
+      return [digest, md5.hexdigest, Base64.encode64(digest).strip]
+    end
+
+    def isZip( filename)
+      f = File.open(filename, 'r')
+      header = f.read(4)
+      return header == 'PK'+3.chr+4.chr
+    end
+
+    def logdebug( s)
+      if @debug
+        print s.to_s
+      end
+    end
+
+    def error( message)
+      logdebug(message)
+      raise StandardError, message
+    end
+
+    def answer_error( answer, message)
+      raise StandardError, sprintf( "%s\n%s", message,  answer['body'])
+    end
+
+    # sendcallback is an object with
+    #  - a 'sendCallBack' member function that accept a unique int argument (=number of bytes written so far)
+    #  - a 'sendBlockSize' member function with no argument which return the size of block to be sent
+    def sendContent( method, url, contentType, filename = nil, body = nil,  parameters = nil, sendcallback = nil)
+
+      # SEND DATA
+      conn = connectionGet()
+
+      md5, md5hex, md5base64 = md5FileOrBody(filename, body)
+
+      if filename
+        size = File.stat(filename).size
+      else
+        size = body.length
+      end
+
+      headers = {'Content-MD5' => md5base64.to_s,
+                 'Content-Length' => size.to_s,
+                 'Content-Type' => contentType}
+
+      url = signUrl(url, method, md5base64, contentType, parameters)
+
+      # LAUNCH THE REQUEST : TODO : pass filename instead of body
+      if method == "PUT"
+        answer = conn.request_put(url, args = nil, body = body, filename = filename, headers = headers, sendcallback = sendcallback)
+      elsif method == "POST"
+        answer = conn.request_post(url,  args = nil, body = body, filename = filename, headers = headers)
+      elsif method == "DELETE"
+        answer = conn.request_delete(url, headers = headers)
+      end
+
+      headers = answer['headers']
+
+      logdebug(headers)
+      logdebug(answer['body'])
+
+      # NOW CHECK THAT EVERYTHING IS OK
+      status = headers['status']
+      if status != '200'
+        msg = sprintf( "sendContent : bad STATUS %s", status )
+        answer_error(answer, msg)
+      end
+
+      if headers['etag'] == nil
+        msg = "corrupted answer: no etag in headers. Response body is " + answer['body']
+        error(msg)
+      end
+
+      obtainedMD5 = headers['etag']
+
+      if obtainedMD5 != md5hex
+        msg = sprintf( "sendContent : bad returned etags %s =! %s (ref)", obtainedMD5, md5hex)
+        error(msg)
+      end
+
+      return answer
+    end
+
+    def getContentUrl( url, method, parameters)
+      return signUrl(url, method, "", "", parameters)
+    end
+
+    def getContent( url, filename = nil, parameters = nil)
+      sleepTime = @sleepTime
+
+      for i in 0..@maxRetry
+        raiseExceptionOn404 = (i + 1) == @maxRetry
+        ret = getContent_(url, filename, parameters, raiseExceptionOn404)
+        if ret["status"] != 404
+          return ret
+        end
+        # Wait for amazon S3 ...
+        sleep(1)
+      end
+    end
+
+    def getContent_( url, filename = nil, parameters = nil, raiseExceptionOn404 = true)
+      method = "GET"
+      url = getContentUrl(url, method, parameters)
+
+      # GET DATA
+      conn = connectionGet()
+      answer = conn.request_get(url)
+      body = answer['body']
+
+      headers = answer['headers']
+      status = headers['status'].to_i
+
+      if status == 204
+        # there was no content
+        obtainedSize = 0
+        if body.length != 0
+          error("204 status with non empty body.")
+        end
+      elsif status == 200
+        obtainedSize =headers['content-length'].to_i
+      elsif status == 404 and not raiseExceptionOn404
+        return  {"url" => headers['content-location'], "status" => 404}
+      else
+        msg = sprintf( "getContent : bad STATUS %s", status )
+        answer_error(answer, msg)
+      end
+
+      if body.length != obtainedSize
+        error("Non matching body length and content-length")
+      end
+
+      if filename != nil
+        f = File.open(filename, 'w')
+        f.write(body)
+        f.close
+
+        if obtainedSize == 0
+          File.unlink(filename)
+        else
+          filesize = File.stat(filename).size
+          if obtainedSize != filesize
+            File.unlink(filename)
+            error(sprintf( "file size is incorrect : file size = %d, body size = %d", filesize, obtainedSize))
+          end
+        end
+      end
+
+      # NOW CHECK EVERYTHING IS OK
+      md5, md5hex, md5base64 = md5FileOrBody(filename, body)
+
+      if status != 204
+        obtainedMD5 = headers['etag'].gsub(/"/,"")
+        if obtainedMD5 != md5hex
+          if filename
+            File.unlink(filename)
+          end
+          error(sprintf( "getDefinition : bad returned etag %s =! %s (ref)", md5hex, obtainedMD5))
+        end
+      end
+
+      if status == 200
+        logdebug(sprintf( "headers = %s", headers) )
+        url = headers['content-location']
+        ret = {'size' => obtainedSize, 'url' => url, 'headers' =>  headers}
+      else
+        ret = {'size' => obtainedSize, 'url' => url}
+      end
+
+      if not filename
+        ret['body'] = body
+      end
+
+      ret["status"] = status
+
+      return ret
+    end
+  end
+end

data/lib/stupeflixclient/stupeflix_client.rb

@@ -0,0 +1,354 @@
+require 'stupeflixclient/stupeflix_base'
+require 'stupeflixclient/connection'
+require 'cgi'
+require 'json.rb'
+module Stupeflixclient
+  class StupeflixClient < StupeflixBase
+    def initialize(accessKey, privateKey, host = "http://services.stupeflix.com", service = 'stupeflix-1.0', debug = false)
+      super(accessKey, privateKey, host, service, debug)
+      @batch = false
+      @batchData = ""
+    end
+
+    # Start a batch, used for speeduping video definition upload
+    # Operation that can be batched : sendDefinition and createProfiles
+    # Operation
+    # Only works for xml definition, not zip, and xml must be in UTF8
+    def batchStart( maxSize = 1000000)
+      @batch = true
+      @batchData = "<batch>"
+      @batchMaxSize = maxSize
+    end
+
+    # End a batch: actually send data
+    def batchEnd
+      @batchData += "</batch>"
+      sendDefinitionBatch(body = @batchData)
+      @batchData = ""
+      @batch = false
+    end
+
+    # Send a definition file to the API
+    def sendDefinition( user, resource, filename = nil, body = nil)
+      url = definitionUrl(user, resource)
+      if body
+        contentType = @TEXT_XML_CONTENT_TYPE;
+      elsif isZip(filename)
+        contentType = @APPLICATION_ZIP_CONTENT_TYPE
+      else
+        contentType = @TEXT_XML_CONTENT_TYPE
+      end
+      if @batch and contentType == @TEXT_XML_CONTENT_TYPE
+        @batchData += sprintf("<task user=\"%s\" resource=\"%s\">", user, resource)
+        if body
+          @batchData += body
+        else
+          @batchData += File.open(filename).read
+        end
+      else
+
+        return sendContent("PUT", url, contentType, filename, body)
+      end
+    end
+
+    # Send a definition file to the API
+    def sendDefinitionBatch( filename = nil, body = nil)
+      url = @definitionBatchUrl
+      contentType = @TEXT_XML_CONTENT_TYPE;
+      return sendContent("PUT", url, contentType, filename, body)
+    end
+
+    def getDefinition( user, resource, filename)
+      url = definitionUrl(user, resource)
+      return getContent(url, filename)['size']
+    end
+
+    def _getAbsoluteUrl( url, followRedirect = false)
+      urlPart = getContentUrl(url, 'GET', nil)
+      if followRedirect
+        conn = connection.Connection(@base_url, followRedirect = false)
+        response = conn.request_get(urlPart)
+        return response["headers"]["location"]
+      else
+        return @base_url + urlPart
+      end
+    end
+
+    def getProfileUrl( user, resource, profile, followRedirect = false)
+      url = profileUrl(user, resource, profile)
+      return _getAbsoluteUrl(url, followRedirect)
+    end
+
+    def getProfile( user, resource, profile, filename)
+      url = profileUrl(user, resource, profile)
+      getContent(url, filename)
+    end
+
+    def getProfileThumbUrl( user, resource, profile, followRedirect = false)
+      url = profileThumbUrl(user, resource, profile, "thumb.jpg")
+      return _getAbsoluteUrl(url, followRedirect)
+    end
+
+    def getProfileThumb( user, resource, profile, filename)
+      url = profileThumbUrl(user, resource, profile, "thumb.jpg")
+      getContent(url, filename)
+    end
+
+    def getProfileReportUrl( user, resource, profile, followRedirect = false)
+      url = profileReportUrl(user, resource, profile)
+      return _getAbsoluteUrl(url, followRedirect)
+    end
+
+    def getProfileReport( user, resource, profile, filename)
+      url = profileReportUrl(user, resource, profile)
+      getContent(url, filename)
+    end
+
+    def createProfiles( user, resource, profiles)
+      profileData = profiles.xmlGet
+      if @batch
+        @batchData += profileData
+        @batchData += "</task>"
+        if @batchData.length >= @batchMaxSize
+          begin
+            @batchEnd
+            finally
+            batchStart(@batchMaxSize)
+          end
+        end
+      else
+        url, parameters = profileCreateUrl(user, resource, profileData)
+        contentType = @APPLICATION_URLENCODED_CONTENT_TYPE
+        body = ""
+        parameters.each_pair do |k,v|
+          body += CGI::escape(k) + "=" + CGI::escape(v)
+        end
+        return sendContent("POST", url, contentType, filename = nil, body = body)
+      end
+    end
+
+    def getStatus( user = nil, resource = nil, profile = nil, marker = nil, maxKeys = nil)
+      url, parameters = statusUrl(user, resource, profile, marker, maxKeys)
+      ret = getContent(url, filename = nil, parameters = parameters)
+      status = JSON.parse(ret['body'])
+      return status
+    end
+
+    def getMarker( status)
+      if status.length == 0
+        return nil
+      end
+      lastStatus = status[-1]
+      #return map(lambda x: lastStatus[x], ["user", "resource", "profile"])
+      return []
+    end
+
+    # helper functions : build non signed urls for each kind of action
+    def definitionUrl( user, resource)
+      return sprintf( "/%s/%s/definition/", user, resource)
+    end
+
+    # helper functions : build non signed urls for each kind of action
+    def definitionBatchUrl
+      return "/batch/"
+    end
+
+    def profileUrl( user, resource, profile)
+      return sprintf( "/%s/%s/%s/", user, resource, profile)
+    end
+
+    def profileThumbUrl( user, resource, profile, thumbname)
+      return sprintf( "/%s/%s/%s/%s/", user, resource, profile, thumbname)
+    end
+
+    def profileReportUrl( user, resource, profile)
+      return sprintf( "/%s/%s/%s/%s/", user, resource, profile, "report.xml")
+    end
+
+    def profileCreateUrl( user, resource, profiles)
+      s = sprintf( "/%s/%s/", user, resource)
+      parameters = {@XML_PARAMETER => profiles}
+      return s, parameters
+    end
+
+    def actionUrl( user, resource, profile, action)
+      path = [user, resource, profile]
+      s = ""
+      path.each do |p|
+        if p == nil
+          break
+        end
+        s += sprintf( "/%s", p )
+      end
+      s += sprintf( "/%s/", action )
+      return s
+    end
+
+    def statusUrl( user, resource, profile, marker = nil, maxKeys = nil)
+      params = {}
+      if marker != nil
+        params[@MARKER_PARAMETER] = marker.join('/')
+      end
+      if maxKeys != nil
+        params[@MAXKEYS_PARAMETER] = maxKeys
+      end
+
+      return [actionUrl(user, resource, profile, "status"), params]
+    end
+  end
+
+  class StupeflixXMLNode
+    def initialize( nodeName, attributes = nil, children = nil, text = nil)
+      @children = children
+      @attributes = attributes
+      @nodeName = nodeName
+      @text = text
+    end
+
+    def xmlGet
+      docXML = '<' + @nodeName
+      if @attributes and @attributes.length != 0
+
+        @attributes.each_pair do |k, v|
+          docXML += " "
+          if v == nil
+            v = ""
+          end
+          k = k.to_s
+          v = v.to_s
+          docXML += k + '="' + CGI.escapeHTML(v) + '"'
+        end
+      end
+      docXML += '>'
+      if @children
+        for c in @children
+          docXML += c.xmlGet
+        end
+      end
+      if @text
+        docXML += @text
+      end
+      docXML += '</' + @nodeName + '>'
+
+      return docXML
+    end
+
+    def metaChildrenAppend( meta = nil, notify = nil, children = nil)
+      childrenArray = []
+      if meta
+        childrenArray += [meta]
+      end
+      if notify
+        childrenArray += [notify]
+      end
+      if children
+        childrenArray += children
+      end
+      return childrenArray
+    end
+  end
+
+  class StupeflixMeta < StupeflixXMLNode
+    def initialize(dict)
+      children = []
+
+      dict.all? {|k, v|
+        children += [StupeflixXMLNode.new(k, nil, nil, v)]
+      }
+      super("meta", {}, children)
+    end
+  end
+
+  class StupeflixProfileSet < StupeflixXMLNode
+    def initialize( profiles, meta = nil, notify = nil)
+      children = metaChildrenAppend(meta, notify, profiles)
+      super("profiles", {}, children)
+    end
+
+    def deflt(profiles)
+      profSet = []
+      for p in profiles
+        upload = StupeflixDefaultUpload
+        profSet += [StupeflixProfile(p, [upload])]
+      end
+
+      return  StupeflixProfileSet.new(profSet)
+    end
+  end
+
+  class StupeflixProfile < StupeflixXMLNode
+    def initialize( profileName, uploads = nil, meta = nil, notify = nil)
+      children = metaChildrenAppend(meta, notify, uploads)
+      super("profile", {"name" => profileName}, children)
+    end
+  end
+
+  class StupeflixNotify < StupeflixXMLNode
+    def initialize( url, statusRegexp)
+      super("notify", {"url" => url, "statusRegexp" => statusRegexp})
+    end
+  end
+
+  class StupeflixHttpHeader < StupeflixXMLNode
+    def initialize(key, value)
+      super("header", {"key" => key, "value" => value})
+    end
+  end
+
+  class StupeflixUpload < StupeflixXMLNode
+    def initialize( name, parameters, meta = nil, notify = nil, children = nil)
+      children = metaChildrenAppend(meta, notify, children)
+      super(name, parameters, children)
+    end
+  end
+
+  class StupeflixHttpPOSTUpload < StupeflixUpload
+    def initialize( url, meta = nil, notify = nil)
+      super("httpPOST", {"url" => url}, meta, notify)
+    end
+  end
+
+  class StupeflixHttpPUTUpload < StupeflixUpload
+    def initialize( url, meta = nil, notify = nil, headers = nil)
+      super("httpPUT", {"url" => url}, meta, notify, headers)
+    end
+  end
+
+  class StupeflixYoutubeUpload < StupeflixUpload
+    def initialize(login, password, meta = nil, notify = nil)
+      super("youtube", {"login" => login, "password" => password}, meta, notify)
+    end
+  end
+
+  class StupeflixBrightcoveUpload < StupeflixUpload
+    def initialize(token, reference_id = nil, meta = nil, notify = nil)
+      parameters = {"sid" => token}
+      if reference_id != nil
+        parameters["reference_id"] = reference_id
+      end
+      super("brightcove", parameters, meta, notify)
+    end
+  end
+
+  class StupeflixDefaultUpload < StupeflixUpload
+    def initialize( meta = nil, notify = nil)
+      children = metaChildrenAppend(meta)
+      super("stupeflixStore", {}, meta, notify)
+    end
+  end
+
+  class StupeflixS3Upload < StupeflixUpload
+    def initialize(bucket, resourcePrefix, accesskey = nil, secretkey = nil, meta = nil, notify = nil)
+      children = metaChildrenAppend(meta)
+      parameters = {"bucket" => bucket, "resourcePrefix" => resourcePrefix}
+      if accesskey != nil
+        parameters["accesskey"] = accesskey
+      end
+      if secretkey != nil
+        parameters["secretkey"] = secretkey
+      end
+      super("s3", parameters, meta, notify)
+    end
+  end
+
+end