s3_to_drive 0.1.1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b0c426f8242c7448f1b2951ad60d5573b61b92ceb3e2443007c0183121b9befd
-  data.tar.gz: e3f37c4a2ec4931f95fdf63874ea2e171c981b80d2fe0ce196861239c3ccff2d
+  metadata.gz: b7f9841bcacc693525edfb210d91fc4ab55185fcd885efe232e41829146cdb05
+  data.tar.gz: 3ee8e6668a761b6ea1072fa3419cae834bff070662947aee7ecf2df19754d71b
 SHA512:
-  metadata.gz: c83626376f55d2ee3f9bd42595984636bbb500236303e793cef3c10ec7e44dcb1844d79a265939dba222d5f09e042396abc3edcb6500b29b0d3c66cb28d01c90
-  data.tar.gz: 6e88bbef05ce46a322c5993e92a0bc80ac8e2a2d1060a653e8d296c46f14599451b9a5f88b9685aa191d60bee483098c67b1a2d703803f2fca6fe6c894369725
+  metadata.gz: 51450cde1370cb2fa047718ae99b41a9f77a26500d3e9cacb0082a80460f01def891b7335d692d7890866f3b3184d37d54ed880669d2766178f6d38d5c625b4a
+  data.tar.gz: aa8b7a75bc988238f0337da7c86c771267bb01023225fb14b3e7f6d1a79bcdbeba97c786d30d4e76230353da58ac60a2c95b1308f37dc5a16e54d8dd0e8febfe

data/lib/s3_to_drive/client/client.rb

@@ -1,3 +1,9 @@
+require_relative "configuration"
+require_relative "validation"
+require_relative "../concerns/drive_upload"
+require_relative "../concerns/state_manager"
+require_relative "../concerns/drive_uploader"
+
 module S3ToDrive
   module Client
     class Client
@@ -10,13 +16,35 @@ module S3ToDrive
         end
       end
 
-      def transfer
+      def transfer(s3_path, drive_id, folder_name)
+        S3ToDrive.validate!
         puts "Bucket name: #{bucket_name}"
         puts "AWS Region: #{aws_region}"
         puts "AWS Access Key ID: #{aws_access_key_id}"
         puts "AWS Secret Access Key: #{aws_secret_access_key}"
-      end
+        puts "S3 Path: #{s3_path}"
+        puts "Drive ID: #{drive_id}"
+        puts "Folder Name: #{folder_name}"
 
+        begin
+          stream_s3_to_drive(
+            s3_bucket: bucket_name,
+            s3_key: s3_path,
+            drive_folder_id: drive_id,
+            folder_name: folder_name
+          )
+        rescue Interrupt
+          puts "\n\n[INTERRUPTED] Upload interrupted by user"
+          puts "[INFO] State saved. Resume by running the same command again."
+          exit(1)
+        rescue StandardError => e
+          puts "\n[FATAL ERROR] #{e}"
+          puts e.backtrace
+          exit(1)
+        end
+
+        puts "🎉 DONE"
+      end
     end
   end
 end

data/lib/s3_to_drive/client/configuration.rb

@@ -1,33 +1,36 @@
 module S3ToDrive
   module Configuration
-    
     VALID_OPTIONS_KEYS = %i[
       aws_access_key_id
       aws_secret_access_key
       aws_region
       bucket_name
+      service_account_file
     ].freeze
 
     attr_accessor(*VALID_OPTIONS_KEYS)
 
-    # def self.extended(base)
-    #   base.reset
-    # end
-
-    # def reset
-    #   self.aws_access_key_id = '123'
-    #   self.aws_secret_access_key = '123'
-    #   self.aws_region = '123'
-    #   self.bucket_name = '123'
-    #   self
-    # end
-
     def validate!
       missing_keys = VALID_OPTIONS_KEYS.select { |key| send(key).nil? }
-      raise MissingConfigurationError, "Missing configuration: #{missing_keys.join(", ")}" if missing_keys.any?
+      return unless missing_keys.any?
+
+      raise S3ToDrive::MissingConfigurationError,
+            "Missing configuration: #{missing_keys.join(", ")}"
+    end
+
+    def self.extended(base)
+      base.reset
+    end
+
+    def reset
+      self.aws_access_key_id  = "AKIA3VOL6WH7UI2CD4N7"
+      self.aws_secret_access_key = "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"
+      self.aws_region = "ap-northeast-1"
+      self.bucket_name = "kidsly-dev-private-contents"
+      self.service_account_file = "service_account.json"
     end
 
-    # Support S3ToDrive::Configuration.configure { |c| ... }
+    # Support S3ToDrive.configure { |c| ... }
     def configure
       yield self
       self

data/lib/s3_to_drive/client/validation.rb

@@ -0,0 +1,9 @@
+module S3ToDrive
+  module Client
+    module Validation
+      def check_configuration
+        S3ToDrive.validate!
+      end
+    end
+  end
+end

data/lib/s3_to_drive/concerns/drive_upload.rb

@@ -0,0 +1,149 @@
+require 'aws-sdk-s3'
+require 'json'
+require 'time'
+
+require_relative 'state_manager'
+require_relative 'drive_upload'
+
+def stream_s3_to_drive(s3_bucket:, s3_key:, drive_folder_id:, folder_name:)
+  # Tạo file state để lưu trạng thái upload
+  state_mgr = StateManager.new('state.json')
+  # Tạo uploader để upload file lên Google Drive
+  uploader = DriveUpload.new(service_account_file: service_account_file)
+  # Tạo s3 client để upload file lên S3
+  s3_client = Aws::S3::Client.new(
+    region: aws_region,
+    access_key_id: aws_access_key_id,
+    secret_access_key: aws_secret_access_key
+  )
+
+  puts "S3 to Google Drive Transfer"
+  puts "S3 Bucket: #{s3_bucket}"
+  puts "S3 Key: #{s3_key}"
+  puts "Drive Folder: #{drive_folder_id}"
+  puts "Folder Name: #{folder_name}"
+
+  # ===== Get file info =====
+  begin
+    head = s3_client.head_object(bucket: s3_bucket, key: s3_key)
+    file_size = head.content_length
+    filename = File.basename(s3_key)
+
+    puts "File: #{filename}"
+    puts "Size: #{file_size} bytes (#{(file_size.to_f / 1024**3).round(2)} GB)"
+  rescue => e
+    puts "[ERROR] Failed to get S3 file info: #{e}"
+    exit(1)
+  end
+
+  # ===== Load state =====
+  state = state_mgr.load
+
+  if state
+    puts "[RESUME] Found existing upload state"
+    puts "[RESUME] Progress: #{state['uploaded_bytes']} / #{file_size}"
+    puts "[RESUME] #{(state['uploaded_bytes'] * 100.0 / file_size).round(1)}%\n"
+
+    if state['s3_key'] != s3_key || state['file_size'] != file_size
+      puts "[WARNING] State mismatch. Restarting..."
+      state_mgr.delete
+      state = nil
+    end
+  end
+
+  # ===== New upload =====
+  unless state
+    puts "[NEW] Starting new upload...\n"
+
+    begin
+      puts "[DRIVE] Checking folder: #{folder_name}"
+
+      folder_id = uploader.find_folder(
+        folder_name: folder_name,
+        parent_folder_id: drive_folder_id
+      )
+
+      unless folder_id
+        puts "[DRIVE] Creating folder..."
+        folder_id = uploader.create_folder(
+          folder_name: folder_name,
+          parent_folder_id: drive_folder_id
+        )
+      end
+
+      upload_url = uploader.create_resumable_upload(
+        filename: filename,
+        folder_id: folder_id,
+        file_size: file_size
+      )
+
+    rescue => e
+      puts "[ERROR] Failed to init upload: #{e}"
+      exit(1)
+    end
+
+    state = state_mgr.create_initial_state(
+      s3_bucket: s3_bucket,
+      s3_key: s3_key,
+      upload_url: upload_url,
+      file_size: file_size,
+      chunk_size: DriveUpload::CHUNK_SIZE
+    )
+  end
+
+  # ===== Upload loop =====
+  upload_url = state['upload_url']
+  chunk_size = state['chunk_size']
+  offset = state['uploaded_bytes']
+  total_chunks = state['total_chunks']
+
+  start_time = Time.now
+
+  while offset < file_size
+    end_byte = [offset + chunk_size, file_size].min
+    range = "bytes=#{offset}-#{end_byte - 1}"
+
+    puts "[CHUNK] #{offset} - #{end_byte}"
+
+    begin
+      # Read from S3
+      resp = s3_client.get_object(
+        bucket: s3_bucket,
+        key: s3_key,
+        range: range
+      )
+
+      chunk_data = resp.body.read
+
+      # Upload to Drive
+      success, new_offset = uploader.upload_chunk(
+        chunk_data: chunk_data,
+        offset: offset,
+        file_size: file_size,
+        upload_url: upload_url
+      )
+
+      raise "Upload failed" unless success
+
+      offset = new_offset
+      state_mgr.update_progress(offset)
+
+      # Progress
+      elapsed = Time.now - start_time
+      speed = offset / elapsed
+      percent = offset * 100.0 / file_size
+      eta = (file_size - offset) / speed
+
+      puts "✅ #{percent.round(1)}% | #{(speed / 1024**2).round(1)} MB/s | ETA #{(eta / 60).round} min"
+
+    rescue => e
+      puts "[ERROR] #{e}"
+      puts "[INFO] Resume supported"
+      exit(1)
+    end
+  end
+
+  puts "\n✅ UPLOAD COMPLETED!"
+
+  state_mgr.delete
+end

data/lib/s3_to_drive/concerns/drive_uploader.rb

@@ -0,0 +1,198 @@
+# lib/drive_upload.rb
+
+require 'net/http'
+require 'json'
+require 'uri'
+require 'googleauth'
+require 'stringio'
+
+class DriveUpload
+  CHUNK_SIZE = 256 * 1024 * 1024 # 256MB
+  MAX_RETRIES = 5
+  SCOPES = ['https://www.googleapis.com/auth/drive.file']
+
+  def initialize(service_account_file:)
+    @authorizer = Google::Auth::ServiceAccountCredentials.make_creds(
+      json_key_io: File.open(service_account_file),
+      scope: SCOPES
+    )
+  end
+
+  # ===== AUTH =====
+  def access_token
+    @authorizer.fetch_access_token!['access_token']
+  end
+
+  # ===== FIND FOLDER =====
+  def find_folder(folder_name:, parent_folder_id:)
+    token = access_token
+
+    query = "name='#{folder_name}' and mimeType='application/vnd.google-apps.folder' and '#{parent_folder_id}' in parents and trashed=false"
+
+    uri = URI("https://www.googleapis.com/drive/v3/files")
+    uri.query = URI.encode_www_form({
+      q: query,
+      fields: 'files(id,name)',
+      supportsAllDrives: 'true',
+      includeItemsFromAllDrives: 'true'
+    })
+
+    req = Net::HTTP::Get.new(uri)
+    req['Authorization'] = "Bearer #{token}"
+
+    res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) { |http| http.request(req) }
+
+    return nil unless res.code.to_i == 200
+
+    data = JSON.parse(res.body)
+    files = data['files'] || []
+
+    if files.any?
+      folder_id = files.first['id']
+      puts "[DRIVE] Found folder: #{folder_name} (#{folder_id})"
+      return folder_id
+    end
+
+    nil
+  end
+
+  # ===== CREATE FOLDER =====
+  def create_folder(folder_name:, parent_folder_id:)
+    token = access_token
+
+    uri = URI("https://www.googleapis.com/drive/v3/files?supportsAllDrives=true")
+
+    body = {
+      name: folder_name,
+      mimeType: 'application/vnd.google-apps.folder',
+      parents: [parent_folder_id]
+    }
+
+    req = Net::HTTP::Post.new(uri)
+    req['Authorization'] = "Bearer #{token}"
+    req['Content-Type'] = 'application/json'
+    req.body = body.to_json
+
+    res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) { |http| http.request(req) }
+
+    unless [200, 201].include?(res.code.to_i)
+      raise "Create folder failed: #{res.code} #{res.body}"
+    end
+
+    data = JSON.parse(res.body)
+    folder_id = data['id']
+
+    puts "[DRIVE] Created folder: #{folder_name} (#{folder_id})"
+    folder_id
+  end
+
+  # ===== CREATE RESUMABLE UPLOAD =====
+  def create_resumable_upload(filename:, folder_id:, file_size:)
+    token = access_token
+
+    uri = URI("https://www.googleapis.com/upload/drive/v3/files?uploadType=resumable&supportsAllDrives=true")
+
+    metadata = {
+      name: filename,
+      parents: [folder_id]
+    }
+
+    req = Net::HTTP::Post.new(uri)
+    req['Authorization'] = "Bearer #{token}"
+    req['Content-Type'] = 'application/json; charset=UTF-8'
+    req['X-Upload-Content-Type'] = 'application/octet-stream'
+    req['X-Upload-Content-Length'] = file_size.to_s
+    req.body = metadata.to_json
+
+    res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) { |http| http.request(req) }
+
+    unless res.code.to_i == 200
+      raise "Create upload session failed: #{res.code} #{res.body}"
+    end
+
+    upload_url = res['Location']
+    raise "No upload URL" unless upload_url
+
+    puts "[DRIVE] Resumable session created"
+    puts "[DRIVE] Upload URL: #{upload_url[0..80]}..."
+
+    upload_url
+  end
+
+  # ===== GET UPLOAD STATUS =====
+  def get_upload_status(upload_url:, file_size:)
+    uri = URI(upload_url)
+
+    req = Net::HTTP::Put.new(uri)
+    req['Content-Range'] = "bytes */#{file_size}"
+    req['Content-Length'] = '0'
+
+    res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) { |http| http.request(req) }
+
+    if res.code.to_i == 308
+      range = res['Range']
+      if range
+        uploaded = range.split('-').last.to_i + 1
+        puts "[DRIVE] Uploaded: #{uploaded}"
+        return uploaded
+      end
+    end
+
+    0
+  end
+
+  # ===== UPLOAD CHUNK =====
+  def upload_chunk(chunk_data:, offset:, file_size:, upload_url:)
+    chunk_size = chunk_data.bytesize
+    end_byte = offset + chunk_size - 1
+
+    uri = URI(upload_url)
+
+    MAX_RETRIES.times do |attempt|
+      begin
+        req = Net::HTTP::Put.new(uri)
+        req['Content-Range'] = "bytes #{offset}-#{end_byte}/#{file_size}"
+        req['Content-Length'] = chunk_size.to_s
+        req.body = chunk_data
+
+        res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true, read_timeout: 300) do |http|
+          http.request(req)
+        end
+
+        case res.code.to_i
+        when 200, 201
+          puts "[DRIVE] ✅ Upload complete"
+          return [true, file_size]
+
+        when 308
+          puts "[DRIVE] ✅ Chunk uploaded #{offset} - #{end_byte}"
+          return [true, end_byte + 1]
+
+        when 429
+          wait = [2**attempt, 60].min
+          puts "[429] Retry #{attempt + 1}/#{MAX_RETRIES}, wait #{wait}s"
+          sleep(wait)
+          next
+
+        else
+          puts "[ERROR] HTTP #{res.code}"
+          puts res.body[0..200]
+          raise "Upload failed"
+        end
+
+      rescue => e
+        puts "[ERROR] #{e}"
+
+        if attempt < MAX_RETRIES - 1
+          wait = [2**attempt, 10].min
+          puts "[RETRY] #{attempt + 1}/#{MAX_RETRIES}, wait #{wait}s"
+          sleep(wait)
+        else
+          raise
+        end
+      end
+    end
+
+    raise "Max retries exceeded at offset #{offset}"
+  end
+end

data/lib/s3_to_drive/concerns/state_manager.rb

@@ -0,0 +1,96 @@
+# lib/s3_to_drive/concerns/state_manager.rb
+
+require 'json'
+require 'time'
+require 'fileutils'
+
+class StateManager
+  attr_reader :state_file, :state
+
+  def initialize(state_file = 'state.json')
+    @state_file = state_file
+    @state = nil
+  end
+
+  # Load state from file
+  def load
+    return nil unless File.exist?(state_file)
+
+    begin
+      @state = JSON.parse(File.read(state_file))
+
+      required_fields = %w[upload_url file_size uploaded_bytes s3_key]
+      unless required_fields.all? { |f| @state.key?(f) }
+        puts "[WARNING] Invalid state file, missing fields. Starting fresh..."
+        return nil
+      end
+
+      puts "[STATE] Loaded: #{@state['uploaded_bytes']} / #{@state['file_size']} bytes"
+      @state
+
+    rescue JSON::ParserError, IOError => e
+      puts "[WARNING] Failed to load state: #{e}. Starting fresh..."
+      nil
+    end
+  end
+
+  # Save state (atomic write)
+  def save(state)
+    @state = state
+    @state['last_update'] = Time.now.utc.iso8601
+
+    temp_file = "#{state_file}.tmp"
+
+    begin
+      File.open(temp_file, 'w:utf-8') do |f|
+        f.write(JSON.pretty_generate(@state))
+      end
+
+      # atomic replace
+      FileUtils.rm_f(state_file)
+      FileUtils.mv(temp_file, state_file)
+
+    rescue IOError => e
+      puts "[ERROR] Failed to save state: #{e}"
+      raise
+    end
+  end
+
+  # Delete state file
+  def delete
+    if File.exist?(state_file)
+      File.delete(state_file)
+      puts "[STATE] Deleted #{state_file}"
+    end
+  end
+
+  # Create initial state
+  def create_initial_state(s3_bucket:, s3_key:, upload_url:, file_size:, chunk_size:)
+    puts "[DRIVE] Upload URL: #{upload_url}"
+
+    state = {
+      's3_bucket' => s3_bucket,
+      's3_key' => s3_key,
+      'upload_url' => upload_url,
+      'file_size' => file_size,
+      'uploaded_bytes' => 0,
+      'chunk_size' => chunk_size,
+      'total_chunks' => (file_size + chunk_size - 1) / chunk_size,
+      'last_chunk_uploaded' => -1,
+      'started_at' => Time.now.utc.iso8601,
+      'last_update' => Time.now.utc.iso8601
+    }
+
+    save(state)
+    state
+  end
+
+  # Update progress
+  def update_progress(uploaded_bytes, chunk_index)
+    raise "State not initialized" if @state.nil?
+
+    @state['uploaded_bytes'] = uploaded_bytes
+    @state['last_chunk_uploaded'] = chunk_index
+    save(@state)
+  end
+end

data/lib/s3_to_drive/exception/missing_config_error.rb

@@ -1,33 +1,3 @@
-# module MissingConfigError
-#   class MissingConfigurationError < StandardError; end
-#   class << self
-#     attr_accessor :config
-
-#     def validate!
-#       missing = REQUIRED_KEYS.select do |key|
-#         config.send(key).nil?
-#       end
-
-#       return if missing.empty?
-
-#       raise MissingConfigurationError, <<~MSG
-#         Missing configuration for MyGem: #{missing.join(", ")}
-
-#         👉 Run:
-#           rails g my_gem:install
-
-#         Then edit:
-#           config/initializers/my_gem.rb
-#       MSG
-#     end
-
-#     def configure
-#       config ||= OpenStruct.new
-#       yield(config)
-#     end
-
-#     def config
-#       @config ||= OpenStruct.new
-#     end
-#   end
-# end
+module S3ToDrive
+  class MissingConfigurationError < StandardError; end
+end

data/lib/s3_to_drive/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module S3ToDrive
-  VERSION = "0.1.1"
+  VERSION = "1.0.0"
 end

data/lib/s3_to_drive.rb

@@ -1,18 +1,19 @@
 # frozen_string_literal: true
 
 require_relative "s3_to_drive/version"
+require_relative "s3_to_drive/exception/missing_config_error"
 require_relative "s3_to_drive/client/configuration"
-require_relative "s3_to_drive/client/client"
 
 module S3ToDrive
   extend Configuration
-  
+
   def self.new
     S3ToDrive::Client::Client.new
   end
 
   def self.method_missing(method_name, *args, &block)
     return super unless new.respond_to?(method_name)
+
     new.send(method_name, *args, &block)
   end
 
@@ -20,3 +21,5 @@ module S3ToDrive
     new.respond_to?(method_name, include_private) || super
   end
 end
+
+require_relative "s3_to_drive/client/client"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: s3_to_drive
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 1.0.0
 platform: ruby
 authors:
 - mirabo-hoang-lx
@@ -48,27 +48,15 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
-- config/initializers/ s3_to_drive.rb
-- lib/concerns/transfer_s3_to_drive/.env
-- lib/concerns/transfer_s3_to_drive/Dockerfile
-- 'lib/concerns/transfer_s3_to_drive/app/__init__.py  '
-- lib/concerns/transfer_s3_to_drive/app/__pycache__/main.cpython-312.pyc
-- lib/concerns/transfer_s3_to_drive/app/config.py
-- lib/concerns/transfer_s3_to_drive/app/db.py
-- lib/concerns/transfer_s3_to_drive/app/drive_upload.py
-- lib/concerns/transfer_s3_to_drive/app/drive_uploader.py
-- lib/concerns/transfer_s3_to_drive/app/kidsly-admin.code-workspace
-- lib/concerns/transfer_s3_to_drive/app/main.py
-- lib/concerns/transfer_s3_to_drive/app/models.py
-- lib/concerns/transfer_s3_to_drive/app/state_manager.py
-- lib/concerns/transfer_s3_to_drive/docker-compose.yml
-- lib/concerns/transfer_s3_to_drive/requirements.txt
-- lib/concerns/transfer_s3_to_drive/service_account.json
 - lib/generators/s3_to_drive/install_generator.rb
 - lib/generators/s3_to_drive/templates/s3_to_drive.rb
 - lib/s3_to_drive.rb
 - lib/s3_to_drive/client/client.rb
 - lib/s3_to_drive/client/configuration.rb
+- lib/s3_to_drive/client/validation.rb
+- lib/s3_to_drive/concerns/drive_upload.rb
+- lib/s3_to_drive/concerns/drive_uploader.rb
+- lib/s3_to_drive/concerns/state_manager.rb
 - lib/s3_to_drive/exception/missing_config_error.rb
 - lib/s3_to_drive/railtie.rb
 - lib/s3_to_drive/version.rb

data/config/initializers/ s3_to_drive.rb

@@ -1,6 +0,0 @@
-S3ToDrive.configure do |config|
-  config.aws_access_key_id = "AKIA26557Y2Q66666666"
-  config.aws_secret_access_key = "AKIA26557Y2Q66666666"
-  config.aws_region = "AKIA26557Y2Q66666666"
-  config.bucket_name = "AKIA26557Y2Q66666666"
-end