flatiron-video-uploader 0.0.14 → 0.0.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9f43e7133ae6b290c0162b2461ff59bc625e57a4
-  data.tar.gz: 72f2a10f1f83d54c99645a940c6f6d970f3fe880
+  metadata.gz: cbf49b7dad7ca4f46930c7cee3a4e412bebeba79
+  data.tar.gz: a543bc4193426d29add0b8cdbd0aad803db531db
 SHA512:
-  metadata.gz: b252d295d0356610f1922ef1dc211cb12415fa5502b3f3b86764c2519fc5f70147bc0422a857ca44951168665f2d1922d2d0d01385ae2067e2f945620e135370
-  data.tar.gz: 7c4d2a49d288740f11c5e086e4b78f0e8565c9d3b6e9e2c166df352093fc7027fa69394172927c7c6cb78c73153dfccd1ca376c8f3f14faed55cce50bca1a066
+  metadata.gz: db45b8fcb6248657b56be6e8a6f0269100846a43cd31f64ffdb422e29b57d4e7edbe1b98312db71ec20dcc700c5033d1bf73506b5f5e37eed025075affe8c9a4
+  data.tar.gz: 81e34fefc754cac995134160804437dc3470b20932ded44e75810255b37de9f851879d44927fa30f5eaa83036ca8e3fc557bf985ceedbda86fbbf34b0b02dddd

data/lib/flatiron-video-uploader/runner.rb

@@ -1,3 +1,39 @@
+PART_SIZE=1024*1024*50
+class File
+  def each_part(current_part=1,part_size=PART_SIZE)
+    (current_part-1).times do 
+      read(part_size)
+    end
+    yield read(part_size) until eof?
+  end
+end
+
+def mark_as_complete(s3,input_opts,lock_hash)
+  input_opts = input_opts.merge({
+    :upload_id   => lock_hash["upload_id"]
+  })   
+
+  parts_resp = s3.list_parts(input_opts)
+
+  input_opts = input_opts.merge(
+    :multipart_upload => {
+      :parts =>
+      parts_resp.parts.map do |part|
+        { :part_number => part.part_number,
+          :etag        => part.etag }
+      end 
+    }   
+  )   
+
+  begin
+    tries ||=3
+    mpu_complete_response = s3.complete_multipart_upload(input_opts)
+    return mpu_complete_response
+  rescue Exception => e
+    retry unless (tries-=1).zero?
+  end
+end
+
 Faraday.default_adapter = :httpclient
 module FlatironVideoUploader
   class Runner 
@@ -125,7 +161,7 @@ module FlatironVideoUploader
       end
       youtube = client.discovered_api("youtube","v3")
 
-      s3 = Aws::S3::Resource.new(
+      s3 = Aws::S3::Client.new(
         # credentials: Aws::Credentials.new('AKIAJY4YSCUKNRWWASIA', 'lXp+rsDPaHcoaEay3XcjfAnE1l1W8BRWOrdLfsPH'),
          credentials: Aws::Credentials.new(token_response['amazon_s3_key'], token_response['amazon_s3_secret']),
         region: 'us-east-1'
@@ -144,33 +180,37 @@ module FlatironVideoUploader
         }
 
         puts "\n\nOK, about to upload the video. This will take a while"
-        video_to_upload = Google::APIClient::UploadIO.new(file, 'video/*')
-
-        video_to_upload.chunk_size =10000000
-        progressbar = ProgressBar.create(:title => "Youtube Upload", :total =>video_to_upload.length)
-        videos_insert_response = client.execute(
-          :api_method => youtube.videos.insert,
-          :body_object => body,
-          :media => video_to_upload,
-          :retries => 0,
-          :parameters => {
-            :uploadType => 'resumable',
-            :part => body.keys.join(','),
-          }
-        )
-        while !videos_insert_response.resumable_upload.complete? do
-          progressbar.progress += video_to_upload.chunk_size
-          videos_insert_response = client.execute(videos_insert_response.resumable_upload)
+        begin
+          video_to_upload = Google::APIClient::UploadIO.new(file, 'video/*')
+
+          video_to_upload.chunk_size =10000000
+          progressbar = ProgressBar.create(:title => "Youtube Upload", :total =>video_to_upload.length)
+          videos_insert_response = client.execute(
+            :api_method => youtube.videos.insert,
+            :body_object => body,
+            :media => video_to_upload,
+            :retries => 0,
+            :parameters => {
+              :uploadType => 'resumable',
+              :part => body.keys.join(','),
+            }
+          )
+          while !videos_insert_response.resumable_upload.complete? do
+            progressbar.progress += video_to_upload.chunk_size
+            videos_insert_response = client.execute(videos_insert_response.resumable_upload)
+          end
+          progressbar.finish
+          puts "Uploaded wooooooooo. The id is #{videos_insert_response.data.id}"
+          puts "\n\n"
+          puts "#"*30
+          puts "Place this in todays schedule on github\n\n\n"
+          command = "<iframe width=\"560\" height=\"315\" src=\"https://www.youtube.com/embed/#{videos_insert_response.data.id}?rel=0&modestbranding=1\" frameborder=\"0\" allowfullscreen></iframe><p><a href=\"https://www.youtube.com/watch?v=#{videos_insert_response.data.id}\">#{video_title}</a></p>"
+          puts command
+          puts "\n\n"
+          puts "#"*30
+        rescue Exception => e
+          exit
         end
-        progressbar.finish
-        puts "Uploaded wooooooooo. The id is #{videos_insert_response.data.id}"
-        puts "\n\n"
-        puts "#"*30
-        puts "Place this in todays schedule on github\n\n\n"
-        command = "<iframe width=\"560\" height=\"315\" src=\"https://www.youtube.com/embed/#{videos_insert_response.data.id}?rel=0&modestbranding=1\" frameborder=\"0\" allowfullscreen></iframe><p><a href=\"https://www.youtube.com/watch?v=#{videos_insert_response.data.id}\">#{video_title}</a></p>"
-        puts command
-        puts "\n\n"
-        puts "#"*30
         TerminalNotifier.notify("Finished Uploading Video to Youtube. Click to copy embed ", :title => "Video Uploader", :execute => `echo '#{command}' | pbcopy`)
       # rescue Google::APIClient::TransmissionError => e
       #   binding.pry
@@ -189,9 +229,113 @@ module FlatironVideoUploader
       movie_transcode_thread.join
 
       puts "\nDone transcoding. Let's upload to S3"
+      bucket = 'flatiron-school-learn-videos'
+      key = "lecture-videos/#{videos_insert_response.data.id}.mp4"
+      lock_file = File.open("/tmp/flatiron-uploader.lock","a+")
+      lock_file.sync=true
+      exit unless lock_file.flock( File::LOCK_NB | File::LOCK_EX )
+
+      file_name = "/tmp/transcoded.mp4"
+      File.open('/tmp/transcoded.mp4', 'rb') do |file|
+        if file.size > PART_SIZE
+          puts "File size over #{PART_SIZE} bytes, using multipart upload..."
+          input_opts = {
+            bucket: bucket,
+            key:    key,
+          }  
+          input_opts_with_acl = input_opts.merge({acl: "public-read"})
+          lock_hash = {}
+          total_parts = file.size.to_f / PART_SIZE
+          current_part=1
+          begin
+            existing_file = lock_file.read
+            lock_hash = JSON.parse existing_file
+            if lock_hash["file_name"]==file_name && lock_hash["file_size"]==file.size && lock_hash["key"]==key && lock_hash["total_parts"] == total_parts
+              binding.pry
+              current_part = lock_hash["current_part"]
+            else
+              input_opts_with_upload = input_opts.merge({
+                :upload_id   => lock_hash["upload_id"],
+                :key => lock_hash["key"],
+              })   
+              s3.abort_multipart_upload(input_opts_with_upload)
+              binding.pry
+              lock_file.truncate(0)
+              mpu_create_response = s3.create_multipart_upload(input_opts_with_acl)
+              lock_hash = {"total_parts"=> total_parts,
+                           "current_part" => current_part,
+                           "file_name" =>file_name,
+                           "file_size" => file.size,
+                           "key" => key,
+                           "upload_id" =>mpu_create_response.upload_id,
+              }
+              lock_file.puts lock_hash.to_json
+            end
+          rescue StandardError => e
+            mpu_create_response = s3.create_multipart_upload(input_opts_with_acl)
+            lock_hash = {"total_parts"=> total_parts,
+                         "current_part" => current_part,
+                         "file_name" =>file_name,
+                         "file_size" => file.size,
+                         "key" => key,
+                         "upload_id" =>mpu_create_response.upload_id,
+            }
+            lock_file.puts lock_hash.to_json
+          end
+          file.each_part(current_part) do |part|
+
+            begin
+              tries ||=3
+              part_response = s3.upload_part({
+                body:        part,
+                bucket:      bucket,
+                key:         key,
+                part_number: current_part,
+                upload_id:   lock_hash["upload_id"],
+              })  
+
+              percent_complete = (current_part.to_f / total_parts.to_f) * 100 
+              percent_complete = 100 if percent_complete > 100 
+              percent_complete = sprintf('%.2f', percent_complete.to_f)
+              puts "percent complete: #{percent_complete}"
+              current_part = current_part + 1 
+              lock_hash["current_part"]=current_part
+              lock_file.truncate(0)
+              lock_file.puts lock_hash.to_json
+            rescue Exception => e
+              if e.class == Interrupt
+                puts "LEaving"
+                lock_file.truncate(0)
+                lock_file.puts lock_hash.to_json
+                File.delete(lock_file.path)
+                exit
+              end
+              puts "trying again"
+              # input_opts = input_opts.merge({
+              #   :upload_id   => mpu_create_response.upload_id,
+              # })   
+              lock_file.truncate(0)
+              lock_file.puts lock_hash.to_json
+
+              retry unless (tries-=1).zero?
+              puts "Couldn't get it to work, exiting"
 
-      obj = s3.bucket('flatiron-school-learn-videos').object("lecture-videos/#{videos_insert_response.data.id}.mp4")
-      obj.upload_file('/tmp/transcoded.mp4', acl:'public-read')
+              File.delete(lock_file.path)
+              exit
+            end
+          end 
+
+          mark_as_complete(s3,input_opts,lock_hash)
+          File.delete(lock_file.path)
+        else
+          s3.put_object(bucket: bucket, key: key, body: file, acl: 'public-read')
+        end
+      end
+
+
+      # obj = s3.bucket('flatiron-school-learn-videos').object("lecture-videos/#{videos_insert_response.data.id}.mp4")
+      # obj.initiate_multipart_upload(options)
+      # obj.upload_file('/tmp/transcoded.mp4', acl:'public-read')
       raw_upload_thread.join unless raw_upload_thread.nil?
       puts "All Done!"
       TerminalNotifier.notify("All Done!", :title =>"Video Uploader")
@@ -199,3 +343,4 @@ module FlatironVideoUploader
     end
   end
 end
+

data/lib/flatiron-video-uploader/version.rb

@@ -1,3 +1,3 @@
 module FlatironVideoUploader
-      VERSION = "0.0.14"
+      VERSION = "0.0.15"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: flatiron-video-uploader
 version: !ruby/object:Gem::Version
-  version: 0.0.14
+  version: 0.0.15
 platform: ruby
 authors:
 - Joe Burgess
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-11 00:00:00.000000000 Z
+date: 2016-02-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler