hls 0.1.0 → 0.2.0

data/lib/hls/uploader.rb

@@ -0,0 +1,211 @@
+# frozen_string_literal: true
+
+require "aws-sdk-s3"
+require "digest"
+require "pathname"
+
+require_relative "instrumentation"
+require_relative "lock"
+
+module HLS
+  # Walks an encoded HLS bundle and pushes each file to the configured
+  # bucket. Idempotent and resumable: a state sidecar tracks per-file
+  # MD5 digests, and files whose remote upload matches the local digest
+  # are skipped.
+  class Uploader
+    CONTENT_TYPES = {
+      ".m3u8" => "application/vnd.apple.mpegurl",
+      ".ts"   => "video/MP2T",
+      ".jpg"  => "image/jpeg",
+      ".jpeg" => "image/jpeg",
+      ".png"  => "image/png",
+      ".vtt"  => "text/vtt"
+    }.freeze
+
+    CACHE_CONTROL_IMMUTABLE = "public, max-age=31536000, immutable"
+
+    # VOD playlists are also immutable once written — segments don't
+    # get rewritten, the playlist itself doesn't change. Use a shorter
+    # max-age than the segments themselves so deploys can publish a
+    # superseding bundle, but allow CDN caching at the playlist edge.
+    CACHE_CONTROL_PLAYLIST = "public, max-age=300"
+
+    # Default retries on top of the AWS SDK's own retry behavior. Bumped
+    # to absorb transient network errors during long, multi-object
+    # uploads where the SDK's retry budget per call has been exhausted.
+    DEFAULT_MAX_RETRIES = 3
+    DEFAULT_INITIAL_BACKOFF = 0.5
+
+    # Parallel upload workers. The SDK is thread-safe, S3 is throughput-
+    # bound on typical connections, and a 3-rendition bundle is dozens
+    # of small segments — overlapping their PUTs cuts wall-clock time
+    # significantly. 4 workers is a good default for home/office links;
+    # bump higher on a beefy server with a fat pipe.
+    DEFAULT_CONCURRENCY = 4
+
+    # Errors classed as transient and worth retrying. We deliberately
+    # don't include the broad Aws::Errors::ServiceError parent — a 403
+    # or NoSuchBucket should fail fast, not retry.
+    TRANSIENT_ERRORS = [
+      Seahorse::Client::NetworkingError,
+      Aws::S3::Errors::RequestTimeout,
+      Aws::S3::Errors::ServiceUnavailable,
+      Aws::S3::Errors::SlowDown,
+      Aws::S3::Errors::InternalError
+    ].freeze
+
+    attr_reader :storage, :output, :key_prefix, :state,
+                :max_retries, :initial_backoff, :concurrency
+
+    def initialize(storage:, output:, key_prefix:, state:,
+                   max_retries: DEFAULT_MAX_RETRIES,
+                   initial_backoff: DEFAULT_INITIAL_BACKOFF,
+                   concurrency: DEFAULT_CONCURRENCY)
+      @storage = storage
+      @output = Pathname.new(output)
+      @key_prefix = key_prefix.to_s.sub(%r{\A/}, "").sub(%r{/\z}, "")
+      @state = state
+      @max_retries = max_retries
+      @initial_backoff = initial_backoff
+      @concurrency = [concurrency.to_i, 1].max
+      @state_mutex = Mutex.new
+    end
+
+    # Upload everything under the output directory that hasn't been
+    # uploaded yet. Returns a hash with :uploaded and :skipped counts.
+    def perform
+      pending = uploadable_files.filter_map do |file|
+        relative_key = relative_key_for(file)
+        digest = md5_of(file)
+        next nil if @state_mutex.synchronize {
+          state.uploaded?(relative_key: relative_key, digest: digest)
+        }
+        [file, relative_key, digest]
+      end
+
+      skipped = uploadable_files.size - pending.size
+      uploaded = upload_in_parallel(pending)
+
+      { uploaded: uploaded, skipped: skipped }
+    end
+
+    private
+
+    def upload_in_parallel(pending)
+      return 0 if pending.empty?
+
+      queue = Queue.new
+      pending.each { |item| queue << item }
+      concurrency.times { queue << :stop }
+
+      uploaded = 0
+      uploaded_mutex = Mutex.new
+      first_error = nil
+      error_mutex = Mutex.new
+
+      workers = Array.new([concurrency, pending.size].min) do
+        Thread.new do
+          loop do
+            item = queue.pop
+            break if item == :stop
+            break if error_mutex.synchronize { first_error }
+            file, relative_key, digest = item
+
+            begin
+              response = upload(file, relative_key: relative_key)
+              @state_mutex.synchronize do
+                state.record_upload(relative_key: relative_key, digest: digest, etag: response.etag)
+                state.save
+              end
+              uploaded_mutex.synchronize { uploaded += 1 }
+            rescue => e
+              error_mutex.synchronize { first_error ||= e }
+              break
+            end
+          end
+        end
+      end
+
+      workers.each(&:join)
+      raise first_error if first_error
+      uploaded
+    end
+
+    def upload(file, relative_key:)
+      key = key_for(relative_key)
+      ct = content_type_for(file)
+      response = nil
+      HLS::Instrumentation.instrument(:upload_object,
+        key: key, bytes: file.size, content_type: ct
+      ) do
+        with_retries(key: key) do
+          object = storage.object(key)
+          response = object.put(
+            body: file.open("rb"),
+            content_type: ct,
+            cache_control: cache_control_for(file)
+          )
+        end
+      end
+      response
+    end
+
+    # Retries a transient error a bounded number of times with
+    # exponential backoff. Per-attempt instrumentation lets the host
+    # app see retries happening (and decide whether the budget is too
+    # generous).
+    def with_retries(key:)
+      attempt = 0
+      backoff = initial_backoff
+      begin
+        yield
+      rescue *TRANSIENT_ERRORS => e
+        attempt += 1
+        raise if attempt > max_retries
+        HLS::Instrumentation.instrument(:upload_retry,
+          key: key, attempt: attempt, error: e.class.name, message: e.message
+        ) {}
+        sleep backoff
+        backoff *= 2
+        retry
+      end
+    end
+
+    def uploadable_files
+      output.glob("**/*")
+        .select { |p| p.file? }
+        .reject { |p| skip?(p) }
+        .sort
+    end
+
+    # Skip the state sidecar and lock file (we never upload them) plus
+    # any junk dotfiles macOS / editors leave behind (.DS_Store, ._*).
+    def skip?(path)
+      basename = path.basename.to_s
+      basename == State::FILENAME ||
+        basename == Lock::FILENAME ||
+        basename == ".DS_Store" ||
+        basename.start_with?("._")
+    end
+
+    def relative_key_for(file)
+      file.relative_path_from(output).to_s
+    end
+
+    def key_for(relative_key)
+      key_prefix.empty? ? relative_key : "#{key_prefix}/#{relative_key}"
+    end
+
+    def md5_of(file)
+      Digest::MD5.file(file).hexdigest
+    end
+
+    def content_type_for(file)
+      CONTENT_TYPES.fetch(file.extname.downcase, "application/octet-stream")
+    end
+
+    def cache_control_for(file)
+      file.extname.downcase == ".m3u8" ? CACHE_CONTROL_PLAYLIST : CACHE_CONTROL_IMMUTABLE
+    end
+  end
+end

data/lib/hls/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HLS
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/hls.rb

@@ -1,212 +1,40 @@
 # frozen_string_literal: true
 
+require "aws-sdk-s3"
+
 require_relative "hls/version"
-require "shellwords"
-require "pathname"
-require "json"
-require "m3u8"
 
 module HLS
   class Error < StandardError; end
 
-  class Poster
-    attr_reader :input, :output, :width, :height
-
-    def initialize(input:, output:, width:, height:)
-      @input = input
-      @output = output.join("poster.jpg")
-      @width = width
-      @height = height
-    end
+  class << self
+    # The Aws::S3::Resource the gem uses when an HLS::Storage::S3
+    # adapter is configured by `bucket_name:` (and resolves the bucket
+    # lazily through this resource). Set in `config/initializers/hls.rb`:
+    #
+    #   HLS.s3_resource = Aws::S3::Resource.new(...)
+    #
+    # Storage::S3 instances may also pass `s3_resource:` directly to
+    # bypass this default.
+    attr_writer :s3_resource
 
-    def command
-      [
-        # invoke ffmpeg
-        "ffmpeg",
-        # overwrite output files without confirmation
-        "-y",
-        # input video file
-        "-i", input,
-        # scale video to target resolution
-        "-vf", "scale=w=#{width}:h=#{height}:force_original_aspect_ratio=decrease",
-        # extract only one frame
-        "-frames:v", "1",
-        # output file path
-        output
-      ]
-    end
-  end
-
-  module Video
-    class Base
-      attr_accessor :input, :output, :renditions
-
-      PLAYLIST = "index.m3u8".freeze
-
-      Rendition = Data.define(:width, :height, :bitrate)
-
-      def initialize(input:, output:)
-        @input = Input.new(input)
-        @output = output
-        @renditions = []
-      end
-
-      def downscaleable_renditions
-        @renditions.select { |r| r.width <= input.width }
-      end
-
-      def rendition(...)
-        @renditions << Rendition.new(...)
-      end
-
-      def command
-        [
-          "ffmpeg",
-          "-y",
-          "-i", @input.path,
-          "-filter_complex", filter_complex
-        ] + \
-        video_maps + \
-        audio_maps + \
-        [
-          "-f", "hls",
-          "-var_stream_map", stream_map,
-          "-master_pl_name", PLAYLIST,
-          "-hls_time", "4",
-          "-hls_playlist_type", "vod",
-          "-hls_segment_filename", segment,
-          playlist
-        ]
-      end
-
-      private
-
-      def filter_complex
-        n = downscaleable_renditions.size
-        split = "[0:v]split=#{n}#{(1..n).map { |i| "[v#{i}]" }.join}"
-        scaled = downscaleable_renditions.each_with_index.map do |rendition, i|
-          "[v#{i + 1}]scale='if(gt(iw,#{rendition.width}),#{rendition.width},iw)':'if(gt(iw,#{rendition.width}),-2,ih)'[v#{i + 1}out]"
-        end
-        ([split] + scaled).join("; ")
-      end
-
-      def video_maps(codec: "h264_videotoolbox")
-        downscaleable_renditions.each_with_index.flat_map do |rendition, i|
-          [
-            "-map", "[v#{i + 1}out]",
-            "-c:v:#{i}", codec,
-            "-b:v:#{i}", "#{rendition.bitrate}k"
-          ]
-        end
-      end
-
-      def audio_maps(codec: "aac", bitrate: 128)
-        downscaleable_renditions.each_with_index.flat_map do |_, i|
-          [
-            "-map", "a:0",
-            "-c:a:#{i}", codec,
-            "-b:a:#{i}", "#{bitrate}k",
-            "-ac", "2"
-          ]
-        end
-      end
-
-      def stream_map
-        downscaleable_renditions.each_index.map { |i| "v:#{i},a:#{i}" }.join(" ")
-      end
-
-      def variant
-        @output.join("%v")
-      end
-
-      def segment
-        variant.join("%d.ts").to_s
-      end
-
-      def playlist
-        variant.join(PLAYLIST).to_s
-      end
-    end
-
-    class Web < Base
-      def initialize(...)
-        super(...)
-        # 360p - Low quality for mobile/slow connections
-        rendition width: 640,  height: 360,  bitrate: 500
-        # 480p - Standard definition for basic streaming
-        rendition width: 854,  height: 480,  bitrate: 1000
-        # 720p - High definition for most desktop viewing
-        rendition width: 1280, height: 720,  bitrate: 3000
-        # 1080p - Full HD for high-quality streaming
-        rendition width: 1920, height: 1080, bitrate: 6000
-        # 4K - Ultra HD for premium viewing experience
-        rendition width: 3840, height: 2160, bitrate: 12000
-      end
-    end
-
-    # Do very little work on vidoes so I can get more dev cycles in.
-    class Dev < Base
-      def initialize(...)
-        super(...)
-        # 360p - Low quality for mobile/slow connections
-        rendition width: 640,  height: 360,  bitrate: 500
-      end
-    end
-  end
-
-  class Input
-    attr_reader :path, :json
-
-    def initialize(path)
-      @path = Pathname(path)
-    end
-
-    def json
-      @json ||= probe
-    end
-
-    def width      = stream.dig(:width)
-    def height     = stream.dig(:height)
-    def codec      = stream.dig(:codec_name)
-    def duration   = json.dig(:format, :duration)&.to_f
-    def framerate  = parse_rate(stream.dig(:r_frame_rate))
-
-    private
-
-    def stream
-      json.dig(:streams, 0)
-    end
-
-    def probe
-      raw = `ffprobe -v error -select_streams v:0 \
-        -show_entries stream=width,height,r_frame_rate,codec_name \
-        -show_entries format=duration \
-        -of json "#{@path}"`
-
-      JSON.parse(raw, symbolize_names: true)
-    end
-
-    def parse_rate(rate)
-      return unless rate
-      num, den = rate.split("/").map(&:to_f)
-      return if den.zero?
-      (num / den).round(3)
-    end
-  end
-
-  class Directory
-    def initialize(source)
-      @source = Pathname.new(source)
-    end
-
-    def glob(glob)
-      Enumerator.new do |y|
-        @source.glob(glob).each do |input|
-          relative = input.relative_path_from(@source)
-          output = relative.dirname.join(relative.basename(input.extname))
-          y << [ input, output ]
-        end
-      end
+    def s3_resource
+      @s3_resource ||= Aws::S3::Resource.new
     end
   end
 end
+
+require_relative "hls/codecs"
+require_relative "hls/instrumentation"
+require_relative "hls/lock"
+require_relative "hls/state"
+require_relative "hls/storage"
+require_relative "hls/cache"
+require_relative "hls/uploader"
+require_relative "hls/manifest"
+require_relative "hls/input"
+require_relative "hls/directory"
+require_relative "hls/application_video"
+
+# Optional Railtie — only loaded when running under Rails.
+require_relative "hls/railtie" if defined?(Rails::Railtie)

data/plans/00-goal.md

@@ -0,0 +1,112 @@
+# Goal
+
+A Rails app declares a video profile class and gets web-playable HLS streaming
+out of a private object store with a single command.
+
+```ruby
+# app/videos/application_video.rb
+class ApplicationVideo < HLS::ApplicationVideo
+  bucket ENV.fetch("VIDEO_S3_BUCKET_NAME")
+  signing_ttl 1.hour
+  segment_duration 4
+end
+
+# app/videos/course_video.rb
+class CourseVideo < ApplicationVideo
+  rendition :full,   scale: 1.0
+  rendition :medium, scale: 0.5
+  rendition :small,  scale: 0.25
+end
+```
+
+```ruby
+# Author drops a file in, kicks off the pipeline:
+CourseVideo.process(input: "lecture.mp4")
+# → probes input
+# → encodes HLS multiplex (multiple renditions + segments + poster)
+# → uploads to Tigris
+# → records sidecar state so re-runs are no-ops
+
+# Browser hits the existing /videos/:id route:
+# → controller asks CourseVideo.manifest(path) for a signed master playlist
+# → m3u8 player streams the variants over pre-signed segment URLs
+```
+
+## What "done" looks like
+
+- [ ] A new mp4 dropped into the input source ends up playable in a browser
+      via the existing `VideosController` with no manual steps.
+      *(Pipeline exists end-to-end; needs a live walk-through against
+      Tigris with a real source file.)*
+- [x] The pipeline is **idempotent**: re-running on an already-processed video
+      uploads nothing.
+- [x] The pipeline is **resumable**: a crashed run resumes without re-encoding.
+- [x] The encode runs on **Linux workers** (Fly.io / CI), not just macOS dev
+      laptops. *(Codec auto-detect is in; CI Linux job not yet wired.)*
+- [x] Preview-window slicing (`VideoPlan#full_video.enabled?` →
+      `variant[0...PREVIEW_DURATION]`) still works for locked content.
+- [x] Twitter-bot poster path (`format.jpeg` → `stream_object`) still works.
+- [ ] The existing live course at `../server` cuts over without breaking
+      currently-playing videos. *(Integrated; needs deploy + smoke test.)*
+
+## The plan
+
+Six steps, each independently verifiable. Cranked in order — earlier steps
+unblock later ones.
+
+- [x] **[01 — Profile DSL](01-profile-dsl.md)**
+      `ApplicationVideo` base + `app/videos/*.rb` autoload via Railtie.
+      Class-level DSL replaces today's imperative `Video::Scalable` /
+      `Video::VTechWatch` constructors.
+- [x] **[02 — Upload pipeline](02-upload-pipeline.md)**
+      `HLS::Uploader` step pushes the bundle to Tigris. Sidecar state file
+      makes it idempotent + resumable. ActiveJob wrapper for queue runners.
+- [x] **[03 — Reader & Manifest](03-reader-and-manifest.md)**
+      Hoist `server/app/models/video.rb` into the gem as `HLS::Manifest`.
+      Profile class becomes the entry point: `CourseVideo.manifest(path)`.
+- [x] **[04 — Codec portability](04-codec-portability.md)**
+      Detect platform and pick `h264_videotoolbox` / `libx264` / `h264_nvenc`.
+      Linux CI proves it.
+- [x] **[05 — Server migration](05-server-migration.md)**
+      Land it all in `../server` behind feature parity with the current
+      `Video` model. Smoke test, cut over, delete the old code.
+- [ ] **[06 — ActiveStorage adapter](06-activestorage-adapter.md)**
+      `has_hls_video :web, profile: WebVideo` on AR models. Attach a
+      source blob → encode job → bundle in dedicated bucket → signed
+      playlists ready to serve. Optional follow-up; the gem works
+      without it.
+
+## Cranking on this
+
+Each plan doc has the same shape:
+
+```
+Goal — one paragraph
+Preconditions — what must already be done
+Work — concrete tasks
+Acceptance — checklist the loop verifies before advancing
+```
+
+Loop driver:
+
+1. Read `00-goal.md` (this file). Find the first unchecked step.
+2. Read that step's plan. Confirm preconditions hold.
+3. Do the work. Run the acceptance checks.
+4. When acceptance passes, tick the step here and move to the next.
+5. Stop when every box at the top is ticked.
+
+Run with:
+
+```
+/loop /implement-next-plan
+```
+
+…or any equivalent driver that re-enters this file each iteration.
+
+## Non-goals (for now)
+
+- Live streaming. VOD only.
+- Per-user DRM beyond pre-signed URLs.
+- Multi-profile fan-out (one source → both `WebVideo` and `MobileVideo`
+  bundles). One profile per attachment; revisit if needed.
+- ActiveStorage integration. The gem owns its own storage layout.

data/plans/01-profile-dsl.md

@@ -0,0 +1,105 @@
+# 01 — Profile DSL
+
+## Goal
+
+A Rails app declares video profiles as classes in `app/videos/`, and each
+class fully describes its renditions, codec, and storage. The profile class is
+the unit — encode, upload, and (later) read all hang off it.
+
+```ruby
+# app/videos/application_video.rb
+class ApplicationVideo < HLS::ApplicationVideo
+  bucket ENV.fetch("VIDEO_S3_BUCKET_NAME")
+  signing_ttl 1.hour
+  segment_duration 4
+end
+
+# app/videos/course_video.rb
+class CourseVideo < ApplicationVideo
+  rendition :full,   scale: 1.0
+  rendition :medium, scale: 0.5
+  rendition :small,  scale: 0.25
+end
+
+# app/videos/screencast_video.rb
+class ScreencastVideo < ApplicationVideo
+  bits_per_pixel :screencast
+  rendition width: 1920, height: 1080, bitrate: 2_500
+  rendition width: 1280, height: 720,  bitrate: 1_500
+end
+```
+
+## Preconditions
+
+- Existing `HLS::Video::Base`, `HLS::Video::Scalable`, `HLS::Video::VTechWatch`
+  classes in `lib/hls.rb` (already there).
+- Existing `HLS::Poster` (already there).
+
+## Work
+
+### 1. Extract `HLS::ApplicationVideo` base class
+
+In `lib/hls/application_video.rb`:
+
+- Class-level DSL: `bucket`, `signing_ttl`, `segment_duration`,
+  `audio_codec`, `audio_bitrate`, `video_codec`, `bits_per_pixel`.
+- Class-level `rendition(...)` accumulator. Two forms:
+  - Explicit: `rendition width:, height:, bitrate:`
+  - Scaled: `rendition :name, scale: 1.0` (computed from input dimensions
+    using the existing `Scalable#estimated_bitrate` math).
+- DSL values are inherited and overridable (subclasses see parent renditions
+  unless they redeclare).
+- `.new(input:, output:)` constructs an instance bound to one input.
+- `#process` runs the full pipeline (encode + poster + upload, wired in
+  step 02).
+
+### 2. Migrate existing classes
+
+- `HLS::Video::Scalable` → keep as a strategy helper (`Scalable.renditions_for(input)`)
+  used by profile classes that want auto-scaling. Or fold it into a
+  `scaled_renditions n: 3` DSL macro on `ApplicationVideo`.
+- `HLS::Video::VTechWatch` → delete (it's a dev-loop hack; replace with a
+  `FastVideo` example profile in tests/fixtures).
+
+### 3. Railtie
+
+In `lib/hls/railtie.rb`:
+
+- Register `app/videos` as an autoload + eager-load path.
+- Expose `Rails.application.config.hls` for app-wide defaults
+  (default bucket, default signing TTL, default codec).
+- Wire `Rails.application.config.hls.s3_client` so the gem uses the host
+  app's `Aws::S3::Resource` configuration (matches `Video::Tigris` setup
+  in `server/app/models/video.rb:4-9`).
+
+### 4. Backwards-compatible shim
+
+Keep `HLS::Video::Base#command` working unchanged so step 02 doesn't have to
+touch ffmpeg arg construction. The DSL classes should compose down to the
+same command array Base produces today.
+
+## Acceptance
+
+- [x] `HLS::ApplicationVideo` exists; `bucket`, `signing_ttl`,
+      `segment_duration`, `rendition` work as documented above.
+- [x] A subclass with three `rendition :name, scale: ...` declarations
+      produces a `command` array byte-identical to today's
+      `HLS::Video::Scalable.new(input:, output:)` for a fixture input.
+- [x] A subclass with explicit `rendition width:, height:, bitrate:`
+      produces the expected ffmpeg command for a fixed-size encode.
+- [x] Subclass inheritance works: a child class inherits parent's `bucket`
+      and `signing_ttl` unless it overrides.
+- [x] Railtie autoloads `app/videos/*.rb` in a Rails dummy app under
+      `spec/dummy` (or equivalent); `CourseVideo` resolves via constantize.
+- [x] `HLS::Video::VTechWatch` is gone; nothing in `examples/` or tests
+      references it.
+- [x] Unit tests cover the DSL accumulator, inheritance, and the
+      Scalable-equivalence golden test.
+
+## Open questions
+
+- Do we want a `scaled_renditions count: 3` macro, or stay explicit per
+  rendition? Probably explicit — it's three lines and more readable.
+- Should `bucket` accept an `Aws::S3::Bucket` directly, a name string, or
+  both? Probably both, with a string being looked up via the configured
+  S3 client.