mp4-rb 0.1.0

data/lib/mp4/source/url.rb

@@ -0,0 +1,130 @@
+require 'net/http'
+require 'uri'
+
+module MP4::Source
+  class Url
+    include Base
+
+    attr_reader :url
+
+    def self.from_uri(uri)
+      unless uri.start_with?('http://') || uri.start_with?('https://')
+        raise ArgumentError, "expected http(s):// URI, got #{uri.inspect}"
+      end
+
+      new(uri)
+    end
+
+    def initialize(url)
+      @url = url
+      @position = 0
+      @eof = false
+      @file_size = fetch_file_size
+    end
+
+    def uri
+      @url
+    end
+
+    def read(n)
+      return '' if eof?
+
+      range_header = "bytes=#{@position}-#{@position + n - 1}"
+      response = http_get(range_header)
+
+      if response.is_a?(Net::HTTPSuccess)
+        data = response.body
+        @position += data.size
+        data
+      else
+        @eof = true
+        ''
+      end
+    end
+
+    def fetch(from, to)
+      range = "bytes=#{from}-#{to}"
+      response = http_get(range)
+
+      if response.is_a?(Net::HTTPSuccess)
+        response.body
+      else
+        raise "Unable to fetch range"
+      end
+    end
+
+    def set_seek(n)
+      @position = n
+      @eof = false if n < @file_size
+    end
+
+    def reset_seek
+      @position = 0
+      @eof = false
+    end
+
+    def open!
+      self
+    end
+
+    def eof?
+      @position >= @file_size
+    end
+
+    def pos
+      @position
+    end
+
+    def fetch_ranges(ranges)
+      range_header = "bytes=#{ranges.join(',')}"
+      response = http_get(range_header)
+
+      if response.is_a?(Net::HTTPSuccess)
+        parse_multipart_response(response.body, response['Content-Type'])
+      else
+        raise "Unable to fetch ranges"
+      end
+    end
+
+    private
+
+    def fetch_file_size
+      uri = URI.parse(@url)
+      request = Net::HTTP::Get.new(uri)
+      request['Range'] = 'bytes=0-0'
+
+      Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http|
+        response = http.request(request)
+        if response.is_a?(Net::HTTPSuccess)
+          content_range = response['Content-Range']
+          if content_range
+            return content_range.split('/').last.to_i
+          else
+            raise "Unable to fetch file size"
+          end
+        else
+          raise "Unable to fetch file size"
+        end
+      end
+    end
+
+    def http_get(range_header)
+      uri = URI.parse(@url)
+      request = Net::HTTP::Get.new(uri)
+      request['Range'] = range_header
+
+      Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https') do |http|
+        http.request(request)
+      end
+    end
+
+    def parse_multipart_response(body, content_type)
+      boundary = content_type.match(/boundary=(.*)$/)[1]
+      parts = body.split("--#{boundary}\r\n").reject { |part| part.strip.empty? || part == "--" }
+      parts.map do |part|
+        _, content = part.split("\r\n\r\n", 2)
+        content.split("\r\n--")[0] # Remove any trailing boundary marker
+      end
+    end
+  end
+end

data/lib/mp4/source.rb

@@ -0,0 +1,23 @@
+module MP4::Source
+  SCHEMES = {
+    'local' => 'MP4::Source::Local',
+    'http'  => 'MP4::Source::Url',
+    'https' => 'MP4::Source::Url',
+    's3'    => 'MP4::Source::S3',
+  }.freeze
+
+  def self.build(uri, base_dir: nil)
+    raise ArgumentError, 'uri must be a String' unless uri.is_a?(String)
+
+    scheme = uri.split(':', 2).first
+    klass_name = SCHEMES[scheme]
+    raise ArgumentError, "unknown source scheme: #{scheme.inspect}" unless klass_name
+
+    klass = Object.const_get(klass_name)
+    if scheme == 'local'
+      klass.from_uri(uri, base_dir: base_dir)
+    else
+      klass.from_uri(uri)
+    end
+  end
+end

data/lib/mp4/structure/service.rb

@@ -0,0 +1,13 @@
+module MP4::Structure
+  class Service
+    attr_reader :mp4
+
+    def initialize(mp4)
+      @mp4 = mp4
+    end
+
+    def next_track_id
+      mp4.cache.next_track_id
+    end
+  end
+end

data/lib/mp4/xlsx/builder.rb

@@ -0,0 +1,44 @@
+module MP4::XLSX
+  module Builder
+    module_function
+
+    def call(mp4:, output_path:)
+      require 'axlsx'
+
+      package = Axlsx::Package.new
+
+      package.workbook do |workbook|
+        mp4.cache.each_track do |track_id|
+          add_track_worksheet(workbook, track_id, mp4)
+        end
+      end
+
+      package.serialize output_path
+    end
+
+    def add_track_worksheet(workbook, track_id, mp4)
+      track = mp4.cache.track(track_id)
+      track_timescale = track.fetch('timescale')
+
+      workbook.add_worksheet name: "trak_#{track_id}" do |sheet|
+        sheet.add_row(['Chunk ID', 'Chunk Offset', 'Sample Number', 'Sample Size', 'Sample Offset', 'Sample Delta', 'Sample C Delta', 'delta sum', 'delta_sum_millsec', 'timestamp', 'Keyframe'])
+
+        mp4.cache.each_sample(track_id) do |row|
+          sheet.add_row([
+                          row['chunk_index'],
+                          row['chunk_offset'],
+                          row['sample_index'],
+                          row['size'],
+                          row['offset'],
+                          row['delta'],
+                          row['c_delta'],
+                          row['delta_sum'],
+                          row['delta_sum_millisec'],
+                          Time.at(row['delta_sum'].to_f / track_timescale).utc.strftime('%H:%M:%S,%L'),
+                          row['keyframe'],
+                        ])
+        end
+      end
+    end
+  end
+end

data/lib/mp4.rb

@@ -0,0 +1,222 @@
+require 'logger'
+require 'stringio'
+require 'bindata'
+require 'nokogiri'
+
+class MP4
+  attr_reader :source, :cache_location, :cache_backend
+
+  LOG_LEVELS = {
+    'silent'  => Logger::UNKNOWN + 1,
+    'error'   => Logger::ERROR,
+    'warn'    => Logger::WARN,
+    'info'    => Logger::INFO,
+    'debug'   => Logger::DEBUG,
+    'verbose' => Logger::DEBUG,
+  }.freeze
+
+  def self.logger
+    @logger ||= begin
+      log = Logger.new($stdout)
+      log.level = LOG_LEVELS.fetch(ENV.fetch('MP4_LOG_LEVEL', 'info').downcase, Logger::INFO)
+      log.formatter = proc do |severity, _time, _prog, msg|
+        "[mp4-rb #{severity.ljust(5)}] #{msg}\n"
+      end
+      log
+    end
+  end
+
+  def self.verbose?
+    ENV.fetch('MP4_LOG_LEVEL', 'info').downcase == 'verbose'
+  end
+
+  def self.root
+    Pathname.new(__dir__).join('..')
+  end
+
+  def initialize(path: nil, url: nil, source: nil, cache_location: nil, cache_backend: nil)
+    @source = source
+    @source ||= MP4::Source::Local.new(path) if path
+    @source ||= MP4::Source::Url.new(url)    if url
+
+    raise 'source is not defined' unless @source
+
+    @cache_location = cache_location
+    @cache_backend  = cache_backend || (cache_location ? :sqlite : :memory)
+
+    return if @cache_location && File.exist?(@cache_location.to_s)
+
+    memoize_mp4
+    segmentize_mp4
+  end
+
+  def build_xslx(output_path)
+    ::MP4::XLSX::Builder.call(mp4: self, output_path: output_path)
+  end
+
+  def cache
+    @cache ||= Cache.build(cache_backend, cache_location)
+  end
+
+  def save_full_to(output_path)
+    t = Layout::Builder.call(cache, cache, source)
+
+    File.open(output_path, 'wb') { |file| file.write(t.generate) }
+  end
+
+  def compute_segments
+    MP4::Nolan::Service.compute_segments(cache)
+  end
+
+  def save_segments_to(output_path)
+    base_path = Pathname.new(output_path)
+
+    FileUtils.mkdir_p(base_path)
+
+    cached_segments = MP4::Nolan::Service.divide_into_segments(cache)
+
+    cached_segments.each_with_index do |cached_part, index|
+      t = Layout::Builder.call(cache, cached_part, source)
+
+      File.open(base_path.join("#{index}.mp4"), 'wb') do |file|
+        file.write(t.generate)
+      end
+    end
+  end
+
+  def build_dash_manifest_xml(init_segment_path:, segment_path:)
+    M4S::Dash::Manifest.generate(cache, init_segment_path: init_segment_path, segment_path: segment_path)
+  end
+
+  def build_dash_init_segment_binary(track_id)
+    M4S::Layout::InitMp4Builder.call(cache, track_id).generate
+  end
+
+  def build_dash_segment_binary(track_id, segment_index)
+    M4S::Layout::Builder.call(cache, source, track_id, segment_index).generate
+  end
+
+  def save_fmp4_segments_to(output_path)
+    base_path = Pathname.new(output_path)
+    FileUtils.mkdir_p(base_path)
+
+    cache.tracks.each_with_index do |track, index|
+      t = M4S::Layout::InitMp4Builder.call(cache, track.fetch('track_id'))
+
+      file_name = "init-#{index}.mp4"
+      file_path = base_path.join(file_name)
+      File.open(file_path, 'wb') { |file| file.write(t.generate) }
+    end
+
+    cache.tracks.each do |track|
+      track_id = track.fetch('track_id')
+
+      cache.segments(track_id).each do |segment|
+        segment_index = segment.fetch('segment_index')
+
+        t = M4S::Layout::Builder.call(cache, source, track_id, segment.fetch('segment_index'))
+        file_name = "chunk-#{track_id.pred}-#{format('%05d', segment_index.next)}.m4s"
+        file_path = base_path.join(file_name)
+
+        MP4.logger.debug { "writing #{file_path}" }
+
+        File.open(file_path, 'wb') { |file| file.write(t.generate) }
+      end
+    end
+
+    File.open(base_path.join('manifest.mpd'), 'w') do |file|
+      file.write(M4S::Dash::Manifest.generate(cache, init_segment_path: nil, segment_path: nil))
+    end
+
+    File.open(base_path.join('general_playlist.m3u8'), 'w') do |file|
+      file.write(M4S::M3U8::GeneralPlaylist.generate(cache))
+    end
+
+    cache.tracks.each do |track|
+      content = M4S::M3U8::MediaPlaylist.generate(cache, track, init_segment_path: nil, segment_path: nil)
+      file_name = "playlist-#{track.fetch('track_id').pred}.m3u8"
+      File.open(base_path.join(file_name), 'w') do |file|
+        file.write(content)
+      end
+    end
+  end
+
+  private
+
+  def memoize_mp4
+    @memoize_mp4 ||= Metadata::Service.call(cache: cache, parsed_mp4: parsed_mp4)
+  end
+
+  def segmentize_mp4
+    @segmentize_mp4 ||= MP4::Nolan::Service.divide_into_segments(cache)
+  end
+
+  def parsed_mp4
+    Parser::Service.new(source: source).call
+  end
+end
+
+require_relative './mp4/ext/refinements'
+require_relative './mp4/constants'
+require_relative './mp4/binary/fixed_point8'
+require_relative './mp4/binary/fixed_point16'
+Dir["#{File.dirname(__FILE__)}/mp4/binary/*.rb"].sort.each {|file| require file }
+Dir["#{File.dirname(__FILE__)}/m4s/binary/*.rb"].sort.each {|file| require file }
+Dir["#{File.dirname(__FILE__)}/m4s/dash/*.rb"].sort.each {|file| require file }
+Dir["#{File.dirname(__FILE__)}/m4s/m3u8/*.rb"].sort.each {|file| require file }
+Dir["#{File.dirname(__FILE__)}/m4s/layout/*.rb"].sort.each {|file| require file }
+require_relative './m4s/local_stream'
+require_relative './mp4/cache'
+require_relative './mp4/cache/sqlite'
+require_relative './mp4/cache/memory'
+require_relative './mp4/source'
+require_relative './mp4/source/base'
+require_relative './mp4/source/local'
+require_relative './mp4/source/url'
+require_relative './mp4/source/s3'
+require_relative './mp4/parser/service'
+require_relative './mp4/parser/moov'
+require_relative './mp4/parser/track'
+require_relative './mp4/parser/main_boxes'
+require_relative './mp4/metadata/service'
+require_relative './mp4/metadata/summary'
+require_relative './mp4/metadata/memoize/ftyp'
+require_relative './mp4/metadata/memoize/tracks'
+require_relative './mp4/metadata/memoize/track'
+require_relative './mp4/metadata/memoize/stbl/chunks'
+require_relative './mp4/metadata/memoize/stbl/sample_to_chunk_calculator'
+require_relative './mp4/metadata/memoize/stbl/composition_offset_calculator'
+require_relative './mp4/metadata/memoize/stbl/delta_calculator'
+require_relative './mp4/metadata/memoize/stbl/samples'
+require_relative './mp4/metadata/memoize/stbl/binary_stsd'
+require_relative './mp4/metadata/memoize/stbl/codec'
+require_relative './mp4/structure/service'
+require_relative './mp4/xlsx/builder'
+require_relative './mp4/nolan'
+require_relative './mp4/nolan/service'
+require_relative './mp4/nolan/service/track'
+require_relative './mp4/nolan/service/sound'
+require_relative './mp4/nolan/service/video'
+require_relative './mp4/layout'
+require_relative './mp4/layout/atom'
+require_relative './mp4/layout/base'
+require_relative './mp4/layout/ftyp'
+require_relative './mp4/layout/free'
+require_relative './mp4/layout/elst'
+require_relative './mp4/layout/mdhd'
+require_relative './mp4/layout/hdlr'
+require_relative './mp4/layout/tkhd'
+require_relative './mp4/layout/mvhd'
+require_relative './mp4/layout/vsmhd'
+require_relative './mp4/layout/dinf'
+require_relative './mp4/layout/stsd'
+require_relative './mp4/layout/stts'
+require_relative './mp4/layout/stss'
+require_relative './mp4/layout/ctts'
+require_relative './mp4/layout/stsc'
+require_relative './mp4/layout/stsz'
+require_relative './mp4/layout/stco'
+require_relative './mp4/layout/mdat'
+require_relative './mp4/layout/container'
+require_relative './mp4/layout/moov'
+require_relative './mp4/layout/builder'

data/mp4-rb.gemspec

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require_relative 'lib/mp4/rb/version'
+
+Gem::Specification.new do |spec|
+  spec.name     = 'mp4-rb'
+  spec.version  = Mp4::Rb::VERSION
+  spec.authors  = ['Edgars Meinarts']
+  spec.email    = ['edgars@meinarts.name']
+
+  spec.summary  = 'Ruby MP4 parser + fMP4 segmenter with DASH and HLS manifest generation.'
+  spec.description = <<~DESC
+    A learning-oriented Ruby library for parsing ISO/IEC 14496-12 MP4 containers
+    and turning them into adaptive-streaming assets. Parses every atom with BinData,
+    memoises the sample table into a pluggable cache (memory or SQLite), cuts each
+    track into keyframe-aligned fMP4 segments, and emits both DASH (MPD) manifests
+    and HLS (m3u8) master + media playlists that reference the same segment bytes.
+    Also includes a full MP4 remux path (regenerated moov from the cache) and an
+    XLSX per-sample analysis workbook. Read the code, run the demo, learn how MP4
+    works under the hood. Not production software.
+  DESC
+
+  spec.homepage = 'https://github.com/edgarMeinart/mp4-rb'
+  spec.license  = 'MIT'
+  spec.required_ruby_version = '>= 2.6.0'
+
+  spec.metadata['homepage_uri']      = spec.homepage
+  spec.metadata['source_code_uri']   = 'https://github.com/edgarMeinart/mp4-rb'
+  spec.metadata['bug_tracker_uri']   = 'https://github.com/edgarMeinart/mp4-rb/issues'
+  spec.metadata['documentation_uri'] = 'https://github.com/edgarMeinart/mp4-rb#readme'
+  spec.metadata['rubygems_mfa_required'] = 'true'
+
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      f.match?(%r{
+        \A(?:
+          # dev-only / build artefacts
+          \.github/|\.rubocop\.yml|\.gitignore|\.rspec|
+          # spec suite
+          spec/|
+          # docker + demo scaffolding (not part of the library)
+          Dockerfile|Dockerfile\.encoder|docker-compose\.yml|Justfile|Makefile|
+          # sinatra-based demo server — excluded per user scope
+          lib/mp4/server/|lib/m4s/local_stream\.rb|
+          # scratch / example scripts and large binary fixtures
+          example\.rb|example_v2\.rb|example_v3\.rb|
+          examples/|migrations/|out/|tmp/|
+          # bootstrap scripts specific to the working repo
+          bin/console|bin/setup|bin/encode\.sh
+        )
+      }x)
+    end
+  end
+
+  spec.bindir      = 'bin'
+  spec.executables = %w[divide_mp4_to_segments.rb build_mp4_xlsx.rb]
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'axlsx',    '~> 2.0.1'
+  spec.add_dependency 'bindata',  '~> 2.4'
+  spec.add_dependency 'nokogiri', '~> 1.15'
+  spec.add_dependency 'sqlite3',  '~> 1.6.0'
+
+  spec.add_development_dependency 'pry',     '~> 0.14'
+  spec.add_development_dependency 'rake',    '~> 13.0'
+  spec.add_development_dependency 'rspec',   '~> 3.12'
+  spec.add_development_dependency 'sinatra', '~> 3.0'
+end

data/sig/mp4/rb.rbs

@@ -0,0 +1,6 @@
+module Mp4
+  module Rb
+    VERSION: String
+    # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+  end
+end

data/todo.txt

@@ -0,0 +1,4 @@
+08.03.2023
+- Calculate duration for TKHD
+- Calculate duration for MDHD
+- Recompile a full video