mp4-rb 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. checksums.yaml +7 -0
  2. data/Gemfile +24 -0
  3. data/Gemfile.lock +149 -0
  4. data/README.md +195 -0
  5. data/Rakefile +8 -0
  6. data/bin/build_mp4_xlsx.rb +35 -0
  7. data/bin/divide_mp4_to_segments.rb +52 -0
  8. data/lib/m4s/binary/ftyp.rb +11 -0
  9. data/lib/m4s/binary/mdat.rb +0 -0
  10. data/lib/m4s/binary/mfhd.rb +18 -0
  11. data/lib/m4s/binary/mvhd.rb +39 -0
  12. data/lib/m4s/binary/sidx.rb +38 -0
  13. data/lib/m4s/binary/styp.rb +17 -0
  14. data/lib/m4s/binary/tfdt.rb +25 -0
  15. data/lib/m4s/binary/tfhd.rb +38 -0
  16. data/lib/m4s/binary/trex.rb +25 -0
  17. data/lib/m4s/binary/trun.rb +30 -0
  18. data/lib/m4s/dash/manifest.rb +149 -0
  19. data/lib/m4s/layout/atom.rb +44 -0
  20. data/lib/m4s/layout/builder.rb +32 -0
  21. data/lib/m4s/layout/container.rb +59 -0
  22. data/lib/m4s/layout/ctts.rb +20 -0
  23. data/lib/m4s/layout/dinf.rb +17 -0
  24. data/lib/m4s/layout/elst.rb +33 -0
  25. data/lib/m4s/layout/ftyp.rb +21 -0
  26. data/lib/m4s/layout/hdlr.rb +22 -0
  27. data/lib/m4s/layout/init_mp4_builder.rb +57 -0
  28. data/lib/m4s/layout/mdat.rb +61 -0
  29. data/lib/m4s/layout/mdhd.rb +25 -0
  30. data/lib/m4s/layout/mfhd.rb +17 -0
  31. data/lib/m4s/layout/mvhd.rb +32 -0
  32. data/lib/m4s/layout/sidx.rb +36 -0
  33. data/lib/m4s/layout/stco.rb +21 -0
  34. data/lib/m4s/layout/stsc.rb +20 -0
  35. data/lib/m4s/layout/stsd.rb +17 -0
  36. data/lib/m4s/layout/stss.rb +20 -0
  37. data/lib/m4s/layout/stsz.rb +19 -0
  38. data/lib/m4s/layout/stts.rb +20 -0
  39. data/lib/m4s/layout/styp.rb +17 -0
  40. data/lib/m4s/layout/tfdt.rb +21 -0
  41. data/lib/m4s/layout/tfhd.rb +17 -0
  42. data/lib/m4s/layout/tkhd.rb +34 -0
  43. data/lib/m4s/layout/trex.rb +33 -0
  44. data/lib/m4s/layout/trun.rb +89 -0
  45. data/lib/m4s/layout/vsmhd.rb +51 -0
  46. data/lib/m4s/m3u8/general_playlist.rb +52 -0
  47. data/lib/m4s/m3u8/media_playlist.rb +32 -0
  48. data/lib/mp4/binary/avc1.rb +98 -0
  49. data/lib/mp4/binary/box_head.rb +8 -0
  50. data/lib/mp4/binary/co64.rb +11 -0
  51. data/lib/mp4/binary/codec.rb +8 -0
  52. data/lib/mp4/binary/ctts.rb +14 -0
  53. data/lib/mp4/binary/ctts_reader.rb +5 -0
  54. data/lib/mp4/binary/dinf.rb +5 -0
  55. data/lib/mp4/binary/dref.rb +5 -0
  56. data/lib/mp4/binary/elst.rb +16 -0
  57. data/lib/mp4/binary/fixed_point16.rb +19 -0
  58. data/lib/mp4/binary/fixed_point8.rb +19 -0
  59. data/lib/mp4/binary/ftyp.rb +9 -0
  60. data/lib/mp4/binary/gmhd.rb +5 -0
  61. data/lib/mp4/binary/hdlr.rb +13 -0
  62. data/lib/mp4/binary/hev1.rb +73 -0
  63. data/lib/mp4/binary/hmhd.rb +14 -0
  64. data/lib/mp4/binary/iods.rb +5 -0
  65. data/lib/mp4/binary/mdhd.rb +18 -0
  66. data/lib/mp4/binary/mp4a.rb +76 -0
  67. data/lib/mp4/binary/mvhd.rb +39 -0
  68. data/lib/mp4/binary/nmhd.rb +8 -0
  69. data/lib/mp4/binary/saio.rb +25 -0
  70. data/lib/mp4/binary/saiz.rb +17 -0
  71. data/lib/mp4/binary/sbgp.rb +5 -0
  72. data/lib/mp4/binary/sdtp.rb +18 -0
  73. data/lib/mp4/binary/senc.rb +12 -0
  74. data/lib/mp4/binary/sgpd.rb +5 -0
  75. data/lib/mp4/binary/smhd.rb +11 -0
  76. data/lib/mp4/binary/stco.rb +13 -0
  77. data/lib/mp4/binary/stco_reader.rb +5 -0
  78. data/lib/mp4/binary/sthd.rb +8 -0
  79. data/lib/mp4/binary/stsc.rb +15 -0
  80. data/lib/mp4/binary/stsc_reader.rb +5 -0
  81. data/lib/mp4/binary/stsd.rb +5 -0
  82. data/lib/mp4/binary/stsd_atom.rb +11 -0
  83. data/lib/mp4/binary/stsd_reader.rb +5 -0
  84. data/lib/mp4/binary/stss.rb +13 -0
  85. data/lib/mp4/binary/stss_reader.rb +5 -0
  86. data/lib/mp4/binary/stsz.rb +18 -0
  87. data/lib/mp4/binary/stsz_reader.rb +5 -0
  88. data/lib/mp4/binary/stts.rb +14 -0
  89. data/lib/mp4/binary/stts_reader.rb +5 -0
  90. data/lib/mp4/binary/tkhd.rb +46 -0
  91. data/lib/mp4/binary/udta.rb +5 -0
  92. data/lib/mp4/binary/vmhd.rb +11 -0
  93. data/lib/mp4/cache/memory.rb +328 -0
  94. data/lib/mp4/cache/sqlite.rb +348 -0
  95. data/lib/mp4/cache.rb +13 -0
  96. data/lib/mp4/constants.rb +87 -0
  97. data/lib/mp4/ext/refinements.rb +19 -0
  98. data/lib/mp4/layout/atom.rb +47 -0
  99. data/lib/mp4/layout/base.rb +10 -0
  100. data/lib/mp4/layout/builder.rb +16 -0
  101. data/lib/mp4/layout/container.rb +59 -0
  102. data/lib/mp4/layout/ctts.rb +35 -0
  103. data/lib/mp4/layout/dinf.rb +15 -0
  104. data/lib/mp4/layout/elst.rb +33 -0
  105. data/lib/mp4/layout/free.rb +20 -0
  106. data/lib/mp4/layout/ftyp.rb +21 -0
  107. data/lib/mp4/layout/hdlr.rb +20 -0
  108. data/lib/mp4/layout/mdat.rb +33 -0
  109. data/lib/mp4/layout/mdhd.rb +27 -0
  110. data/lib/mp4/layout/moov.rb +58 -0
  111. data/lib/mp4/layout/mvhd.rb +46 -0
  112. data/lib/mp4/layout/stco.rb +37 -0
  113. data/lib/mp4/layout/stsc.rb +27 -0
  114. data/lib/mp4/layout/stsd.rb +15 -0
  115. data/lib/mp4/layout/stss.rb +22 -0
  116. data/lib/mp4/layout/stsz.rb +39 -0
  117. data/lib/mp4/layout/stts.rb +35 -0
  118. data/lib/mp4/layout/tkhd.rb +45 -0
  119. data/lib/mp4/layout/vsmhd.rb +49 -0
  120. data/lib/mp4/layout.rb +3 -0
  121. data/lib/mp4/metadata/memoize/ftyp.rb +13 -0
  122. data/lib/mp4/metadata/memoize/stbl/binary_stsd.rb +13 -0
  123. data/lib/mp4/metadata/memoize/stbl/chunks.rb +46 -0
  124. data/lib/mp4/metadata/memoize/stbl/codec.rb +82 -0
  125. data/lib/mp4/metadata/memoize/stbl/composition_offset_calculator.rb +31 -0
  126. data/lib/mp4/metadata/memoize/stbl/delta_calculator.rb +29 -0
  127. data/lib/mp4/metadata/memoize/stbl/sample_to_chunk_calculator.rb +63 -0
  128. data/lib/mp4/metadata/memoize/stbl/samples.rb +80 -0
  129. data/lib/mp4/metadata/memoize/track.rb +98 -0
  130. data/lib/mp4/metadata/memoize/tracks.rb +11 -0
  131. data/lib/mp4/metadata/service.rb +30 -0
  132. data/lib/mp4/metadata/summary.rb +11 -0
  133. data/lib/mp4/nolan/service/sound.rb +36 -0
  134. data/lib/mp4/nolan/service/track.rb +9 -0
  135. data/lib/mp4/nolan/service/video.rb +35 -0
  136. data/lib/mp4/nolan/service.rb +84 -0
  137. data/lib/mp4/nolan.rb +3 -0
  138. data/lib/mp4/parser/main_boxes.rb +43 -0
  139. data/lib/mp4/parser/moov.rb +40 -0
  140. data/lib/mp4/parser/service.rb +47 -0
  141. data/lib/mp4/parser/track.rb +56 -0
  142. data/lib/mp4/rb/version.rb +7 -0
  143. data/lib/mp4/rb.rb +10 -0
  144. data/lib/mp4/source/base.rb +20 -0
  145. data/lib/mp4/source/local.rb +79 -0
  146. data/lib/mp4/source/s3.rb +118 -0
  147. data/lib/mp4/source/url.rb +130 -0
  148. data/lib/mp4/source.rb +23 -0
  149. data/lib/mp4/structure/service.rb +13 -0
  150. data/lib/mp4/xlsx/builder.rb +44 -0
  151. data/lib/mp4.rb +222 -0
  152. data/mp4-rb.gemspec +68 -0
  153. data/sig/mp4/rb.rbs +6 -0
  154. data/todo.txt +4 -0
  155. metadata +323 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: b330d2181372489c047835f18286ab29f5ec410f3e2af0a8c4ef821cd0aaac39
4
+ data.tar.gz: 419e94a9e760e037757f13de0e95f40dc3893b3746dfe8e12d5b5700444221be
5
+ SHA512:
6
+ metadata.gz: e342aceff2bd0cd5fd0dc371ec00059c087f01ee92daf45c6abc589e2fd8f9fa97455f39115bd0ece670ef7c1596fc9246fa9fdbb032caa6b1400d894307ce59
7
+ data.tar.gz: 57e61066f0372834d3fe92c1c356ba4422306ebad64927c69d7402097f249b02a9e3091671ffac43fb6a0df1d0c9df7fee06d0ab1fa6b45f286a9c8a9171c206
data/Gemfile ADDED
@@ -0,0 +1,24 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in mp4-rb.gemspec
4
+ gemspec
5
+
6
+ gem "rake", "~> 13.0"
7
+
8
+ gem "rspec", "~> 3.0"
9
+
10
+ gem 'colorize', '~> 0.8.1'
11
+ gem 'pry'
12
+ gem 'bindata', '~> 2.4', '>= 2.4.14'
13
+ gem 'axlsx', '~> 2.0.1'
14
+ gem 'virtus'
15
+ gem 'sqlite3', '1.6.0'
16
+ gem 'ruby-progressbar', '~> 1.11'
17
+ gem 'sinatra'
18
+ gem 'webrick'
19
+ gem 'nokogiri'
20
+ gem 'aws-sdk-s3', '~> 1.100', require: false
21
+
22
+ group :development, :test do
23
+ gem 'rubocop', '~> 1.60', require: false
24
+ end
data/Gemfile.lock ADDED
@@ -0,0 +1,149 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ mp4-rb (0.1.0)
5
+ axlsx (~> 2.0.1)
6
+ bindata (~> 2.4)
7
+ nokogiri (~> 1.15)
8
+ sqlite3 (~> 1.6.0)
9
+
10
+ GEM
11
+ remote: https://rubygems.org/
12
+ specs:
13
+ ast (2.4.3)
14
+ aws-eventstream (1.4.0)
15
+ aws-partitions (1.1267.0)
16
+ aws-sdk-core (3.253.0)
17
+ aws-eventstream (~> 1, >= 1.3.0)
18
+ aws-partitions (~> 1, >= 1.992.0)
19
+ aws-sigv4 (~> 1.9)
20
+ base64
21
+ bigdecimal
22
+ jmespath (~> 1, >= 1.6.1)
23
+ logger
24
+ aws-sdk-kms (1.129.0)
25
+ aws-sdk-core (~> 3, >= 3.248.0)
26
+ aws-sigv4 (~> 1.5)
27
+ aws-sdk-s3 (1.226.0)
28
+ aws-sdk-core (~> 3, >= 3.248.0)
29
+ aws-sdk-kms (~> 1)
30
+ aws-sigv4 (~> 1.5)
31
+ aws-sigv4 (1.12.1)
32
+ aws-eventstream (~> 1, >= 1.0.2)
33
+ axiom-types (0.1.1)
34
+ descendants_tracker (~> 0.0.4)
35
+ ice_nine (~> 0.11.0)
36
+ thread_safe (~> 0.3, >= 0.3.1)
37
+ axlsx (2.0.1)
38
+ htmlentities (~> 4.3.1)
39
+ nokogiri (>= 1.4.1)
40
+ rubyzip (~> 1.0.0)
41
+ base64 (0.3.0)
42
+ bigdecimal (4.1.2)
43
+ bindata (2.4.14)
44
+ coderay (1.1.3)
45
+ coercible (1.0.0)
46
+ descendants_tracker (~> 0.0.1)
47
+ colorize (0.8.1)
48
+ descendants_tracker (0.0.4)
49
+ thread_safe (~> 0.3, >= 0.3.1)
50
+ diff-lcs (1.5.0)
51
+ htmlentities (4.3.4)
52
+ ice_nine (0.11.2)
53
+ jmespath (1.6.2)
54
+ json (2.20.0)
55
+ language_server-protocol (3.17.0.6)
56
+ lint_roller (1.1.0)
57
+ logger (1.7.0)
58
+ method_source (1.0.0)
59
+ mini_portile2 (2.8.9)
60
+ mustermann (3.0.0)
61
+ ruby2_keywords (~> 0.0.1)
62
+ nokogiri (1.19.1-arm64-darwin)
63
+ racc (~> 1.4)
64
+ parallel (2.1.0)
65
+ parser (3.3.11.1)
66
+ ast (~> 2.4.1)
67
+ racc
68
+ prism (1.9.0)
69
+ pry (0.14.2)
70
+ coderay (~> 1.1)
71
+ method_source (~> 1.0)
72
+ racc (1.6.2)
73
+ rack (2.2.6.4)
74
+ rack-protection (3.0.6)
75
+ rack
76
+ rainbow (3.1.1)
77
+ rake (13.0.6)
78
+ regexp_parser (2.12.0)
79
+ rspec (3.12.0)
80
+ rspec-core (~> 3.12.0)
81
+ rspec-expectations (~> 3.12.0)
82
+ rspec-mocks (~> 3.12.0)
83
+ rspec-core (3.12.1)
84
+ rspec-support (~> 3.12.0)
85
+ rspec-expectations (3.12.2)
86
+ diff-lcs (>= 1.2.0, < 2.0)
87
+ rspec-support (~> 3.12.0)
88
+ rspec-mocks (3.12.3)
89
+ diff-lcs (>= 1.2.0, < 2.0)
90
+ rspec-support (~> 3.12.0)
91
+ rspec-support (3.12.0)
92
+ rubocop (1.88.2)
93
+ json (~> 2.3)
94
+ language_server-protocol (~> 3.17.0.2)
95
+ lint_roller (~> 1.1.0)
96
+ parallel (>= 1.10)
97
+ parser (>= 3.3.0.2)
98
+ rainbow (>= 2.2.2, < 4.0)
99
+ regexp_parser (>= 2.9.3, < 3.0)
100
+ rubocop-ast (>= 1.49.0, < 2.0)
101
+ ruby-progressbar (~> 1.7)
102
+ unicode-display_width (>= 2.4.0, < 4.0)
103
+ rubocop-ast (1.50.0)
104
+ parser (>= 3.3.7.2)
105
+ prism (~> 1.7)
106
+ ruby-progressbar (1.11.0)
107
+ ruby2_keywords (0.0.5)
108
+ rubyzip (1.0.0)
109
+ sinatra (3.0.6)
110
+ mustermann (~> 3.0)
111
+ rack (~> 2.2, >= 2.2.4)
112
+ rack-protection (= 3.0.6)
113
+ tilt (~> 2.0)
114
+ sqlite3 (1.6.0)
115
+ mini_portile2 (~> 2.8.0)
116
+ thread_safe (0.3.6)
117
+ tilt (2.1.0)
118
+ unicode-display_width (3.2.0)
119
+ unicode-emoji (~> 4.1)
120
+ unicode-emoji (4.2.0)
121
+ virtus (2.0.0)
122
+ axiom-types (~> 0.1)
123
+ coercible (~> 1.0)
124
+ descendants_tracker (~> 0.0, >= 0.0.3)
125
+ webrick (1.9.2)
126
+
127
+ PLATFORMS
128
+ arm64-darwin-21
129
+ arm64-darwin-22
130
+
131
+ DEPENDENCIES
132
+ aws-sdk-s3 (~> 1.100)
133
+ axlsx (~> 2.0.1)
134
+ bindata (~> 2.4, >= 2.4.14)
135
+ colorize (~> 0.8.1)
136
+ mp4-rb!
137
+ nokogiri
138
+ pry
139
+ rake (~> 13.0)
140
+ rspec (~> 3.0)
141
+ rubocop (~> 1.60)
142
+ ruby-progressbar (~> 1.11)
143
+ sinatra
144
+ sqlite3 (= 1.6.0)
145
+ virtus
146
+ webrick
147
+
148
+ BUNDLED WITH
149
+ 2.4.6
data/README.md ADDED
@@ -0,0 +1,195 @@
1
+ # mp4-rb
2
+
3
+ > **Ruby MP4 parser + fMP4 segmenter + on-the-fly HLS/DASH origin server.**
4
+ > A learning project — not production software.
5
+
6
+ This project originally started as a personal learning exercise and proof of
7
+ concept to better understand the MP4 container format, including its internal
8
+ structure, parsing, and segmentation.
9
+
10
+ Over time, it evolved into an end-to-end prototype that demonstrates the
11
+ complete media delivery pipeline: parsing MP4 files, generating fragmented
12
+ MP4 (fMP4) segments, and delivering adaptive **HLS** and **DASH** streams.
13
+
14
+ While this implementation is far from production-ready and doesn't reflect
15
+ the architecture of my current streaming server, it showcases the core
16
+ concepts behind MP4 parsing, segment generation, and adaptive streaming.
17
+ It represents an important milestone in my journey toward building a fully
18
+ featured on-the-fly streaming solution.
19
+
20
+ ## ⚠️ Disclaimer
21
+
22
+ This project was built **for fun and for learning purposes**. It is not
23
+ maintained as a product, has known rough edges, and should not be used in
24
+ production. If you're here to understand *how* MP4 containers work under
25
+ the hood — how boxes/atoms nest, how the sample table (`stbl`) drives
26
+ timing, how fragmented MP4 initialization and media segments are laid
27
+ out, or how HLS and DASH manifests reference those segments — this
28
+ repository is a good hands-on reference. You can read the code, run the
29
+ demo, break things, and see what happens.
30
+
31
+ If that sounds useful to you, welcome. If you need a battle-tested
32
+ packager, use [Shaka Packager](https://github.com/shaka-project/shaka-packager)
33
+ or [Bento4](https://www.bento4.com/) instead.
34
+
35
+ ---
36
+
37
+ ## What you can learn from this repo
38
+
39
+ - **How MP4 is parsed.** Every atom is a `BinData::Record` under
40
+ [`lib/mp4/binary/`](lib/mp4/binary/) — `ftyp`, `mvhd`, `tkhd`, `stsd`,
41
+ `stts`, `stsc`, `stsz`, `stco`, `co64`, `ctts`, `elst`, `saio`,
42
+ `saiz`, `senc`, etc. No hand-rolled `unpack`.
43
+ - **How the sample table drives time.** [`lib/mp4/metadata/memoize/`](lib/mp4/metadata/memoize/)
44
+ walks the parsed tree and flattens it into a per-sample cache:
45
+ chunk offset, sample size, decode delta, composition offset,
46
+ keyframe flag, and PTS in milliseconds.
47
+ - **How fMP4 segments are cut.** [`lib/mp4/nolan/`](lib/mp4/nolan/)
48
+ divides each track into segments at video keyframes, and
49
+ [`lib/m4s/`](lib/m4s/) emits the `styp` + `moof` + `mdat` bytes for
50
+ each media segment plus the `moov` init segment.
51
+ - **How HLS + DASH reference the same segments.**
52
+ [`lib/m4s/dash/manifest.rb`](lib/m4s/dash/manifest.rb) writes the MPD
53
+ XML with Nokogiri; [`lib/m4s/m3u8/`](lib/m4s/m3u8/) writes the
54
+ master + media playlists. Both point at the exact same `.m4s` bytes
55
+ (CMAF-style shared init) — the server just relabels the URL suffix.
56
+ - **How to serve any of the above on the fly.**
57
+ [`lib/mp4/server/api.rb`](lib/mp4/server/api.rb) is a Sinatra origin
58
+ that parses on the first request, memoises the result in an LRU cache,
59
+ and serves subsequent segment requests in milliseconds. It reads
60
+ sources from a local directory or from S3-compatible object storage
61
+ (MinIO in the demo) using `GetObject` byte-range reads.
62
+
63
+ ---
64
+
65
+ ## The `just demo` — one-command reviewer setup
66
+
67
+ The fastest way to see the whole pipeline working end-to-end is:
68
+
69
+ ```bash
70
+ just demo
71
+ # open http://127.0.0.1:4567/demo
72
+ ```
73
+
74
+ That single command runs `docker compose up --build` on a four-service
75
+ stack. You need **only Docker** installed — no Ruby, no ffmpeg, no
76
+ bundle install.
77
+
78
+ ### What happens when you run `just demo`
79
+
80
+ 1. **Encoder container** synthesises `examples/adaptive_demo.mp4` — a
81
+ 20-minute video generated from `ffmpeg -f lavfi -i testsrc2` +
82
+ `-f lavfi -i sine` with a `drawtext` HH:MM:SS clock overlay. Four
83
+ video renditions (1080p / 720p / 480p / 360p) + AAC-LC audio,
84
+ GOP=48. No external MP4 required. (If you already have the file, this
85
+ step is skipped.)
86
+ 2. **MinIO container** boots an S3-compatible object store on `:9000`
87
+ (console on `:9001`).
88
+ 3. **MinIO-init container** waits for MinIO to become healthy, creates
89
+ the `videos` bucket, and uploads the fixture with `mc cp`.
90
+ 4. **Server container** waits for the encoder AND the S3 upload to
91
+ complete, then boots the Sinatra origin. Before it accepts a
92
+ single request, it **pre-warms both caches** (local + S3) by
93
+ running the full parse → memoise → segmentise pipeline. This
94
+ takes roughly a minute, and is why the first user hit on
95
+ `manifest.mpd` is ~200 ms instead of 30 seconds.
96
+ 5. Open **http://127.0.0.1:4567/demo** and you get a page with
97
+ **four shaka-player instances** side-by-side:
98
+
99
+ | Card | Endpoint |
100
+ | ---------------- | ------------------------------------------------- |
101
+ | HLS · local | `/local/adaptive_demo.mp4/master.m3u8` |
102
+ | DASH · local | `/local/adaptive_demo.mp4/manifest.mpd` |
103
+ | HLS · S3 (MinIO) | `/s3/videos/adaptive_demo.mp4/master.m3u8` |
104
+ | DASH · S3 (MinIO)| `/s3/videos/adaptive_demo.mp4/manifest.mpd` |
105
+
106
+ Each card has a quality picker (1080p → 360p), an ABR toggle,
107
+ and a live stats overlay (currentTime, buffered, active variant,
108
+ estimated bandwidth, dropped frames). All four play the same
109
+ segments concurrently.
110
+
111
+ Total time from `git clone` to first frame on screen: **~11 minutes**
112
+ on a fresh clone (dominated by the encoder synthesising the 20-minute
113
+ fixture). On subsequent runs, the encoder skips its work and boot
114
+ time drops to under a minute.
115
+
116
+ ### Tearing it down
117
+
118
+ ```bash
119
+ docker compose down # stop containers, keep fixture
120
+ docker compose down -v # also drop MinIO volume
121
+ rm -f examples/adaptive_demo.mp4 # also drop the encoded fixture
122
+ ```
123
+
124
+ ---
125
+
126
+ ## Running things without Docker
127
+
128
+ If you have Ruby 3.x + Bundler + ffmpeg locally, you can also run each
129
+ piece by hand:
130
+
131
+ ```bash
132
+ just install # bundle install
133
+ just spec # run the RSpec suite
134
+ just fixture # encode examples/adaptive_demo.mp4
135
+ just segments examples/foo.mp4 # emit fMP4 + DASH + HLS to out/fmp4/
136
+ just remux examples/foo.mp4 # regenerate a full MP4 from cache
137
+ just xlsx examples/foo.mp4 # per-sample analysis workbook
138
+ just server # boot Sinatra on :4567
139
+ just console # IRB with lib/mp4 loaded
140
+ ```
141
+
142
+ ---
143
+
144
+ ## Repository layout at a glance
145
+
146
+ ```
147
+ lib/mp4/binary/ — BinData readers, one file per atom
148
+ lib/mp4/layout/ — writers for full MP4 remux
149
+ lib/mp4/metadata/ — parsed tree → per-sample cache
150
+ lib/mp4/nolan/ — keyframe-aligned segmentation
151
+ lib/mp4/cache/ — memory + SQLite cache backends
152
+ lib/mp4/source/ — local, HTTP, S3 source adapters
153
+ lib/mp4/server/ — Sinatra origin + LRU source cache
154
+ lib/m4s/binary/ — BinData readers for fMP4 boxes
155
+ lib/m4s/layout/ — init + media segment writers
156
+ lib/m4s/dash/ — MPD XML generation
157
+ lib/m4s/m3u8/ — master + media playlist generation
158
+ ```
159
+
160
+ ---
161
+
162
+ ## Development
163
+
164
+ ```bash
165
+ just install
166
+ just spec
167
+ just lint
168
+ ```
169
+
170
+ `bin/console` (`just console`) drops you into an IRB with `MP4` loaded.
171
+ It's the most useful way to poke at a real file:
172
+
173
+ ```ruby
174
+ mp4 = MP4.new(path: './examples/some.mp4')
175
+ mp4.cache.tracks
176
+ mp4.cache.chunks(1).first
177
+ mp4.build_dash_manifest_xml(init_segment_path: nil, segment_path: nil)
178
+ ```
179
+
180
+ ---
181
+
182
+ ## Not in scope
183
+
184
+ - DRM (Widevine / PlayReady / FairPlay) — parsing of `senc`/`saio`/`saiz`
185
+ is implemented, but nothing is encrypted end-to-end.
186
+ - Live streaming — this is a VOD-only prototype.
187
+ - Production hardening — no authentication, no rate limiting, no CDN
188
+ fronting. Do not expose this to the public internet.
189
+
190
+ ---
191
+
192
+ ## License
193
+
194
+ MIT-ish personal project. Use whatever you learn here freely; blame
195
+ nobody but yourself if you ship it to production.
data/Rakefile ADDED
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "bundler/gem_tasks"
4
+ require "rspec/core/rake_task"
5
+
6
+ RSpec::Core::RakeTask.new(:spec)
7
+
8
+ task default: :spec
@@ -0,0 +1,35 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+ #
4
+ # Emit an XLSX analysis workbook (one sheet per track, per-sample rows) for the
5
+ # given input MP4.
6
+ #
7
+ # Usage:
8
+ # bin/build_mp4_xlsx.rb <input.mp4> [output.xlsx]
9
+ # MP4_INPUT=path.mp4 MP4_OUTPUT=out.xlsx bin/build_mp4_xlsx.rb
10
+
11
+ require 'fileutils'
12
+ require_relative '../lib/mp4'
13
+
14
+ input = ARGV[0] || ENV['MP4_INPUT']
15
+ if input.nil? || input.empty?
16
+ MP4.logger.error 'usage: bin/build_mp4_xlsx.rb <input.mp4> [output.xlsx]'
17
+ exit 1
18
+ end
19
+ unless File.file?(input)
20
+ MP4.logger.error "input not found: #{input}"
21
+ exit 1
22
+ end
23
+
24
+ output = ARGV[1]
25
+ output = ENV['MP4_OUTPUT'] if output.nil? || output.empty?
26
+ if output.nil? || output.empty?
27
+ base = File.basename(input, File.extname(input))
28
+ output = "out/xlsx/#{base}.xlsx"
29
+ end
30
+ FileUtils.mkdir_p(File.dirname(output))
31
+
32
+ MP4.logger.info "Input: #{input}"
33
+ MP4.logger.info "Output: #{output}"
34
+
35
+ MP4.new(path: input).build_xslx(output)
@@ -0,0 +1,52 @@
1
+ #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
3
+ #
4
+ # Emit a full fMP4 DASH/HLS package for the given input MP4:
5
+ # init-N.mp4 (one per track)
6
+ # chunk-{track-1}-NNNNN.m4s (one per segment per track)
7
+ # manifest.mpd (DASH)
8
+ # general_playlist.m3u8 (HLS master)
9
+ # playlist-{track-1}.m3u8 (HLS media per track)
10
+ #
11
+ # Usage:
12
+ # bin/divide_mp4_to_segments.rb <input.mp4> [output_dir]
13
+ # MP4_INPUT=path.mp4 MP4_OUTPUT=out/ bin/divide_mp4_to_segments.rb
14
+ #
15
+ # ENV:
16
+ # MP4_INPUT — input MP4 path (overridden by ARGV[0])
17
+ # MP4_OUTPUT — output directory (overridden by ARGV[1])
18
+ # XLSX — if set, also write an XLSX analysis to out/xlsx/<name>.xlsx
19
+
20
+ require 'fileutils'
21
+ require_relative '../lib/mp4'
22
+
23
+ input = ARGV[0] || ENV['MP4_INPUT']
24
+ if input.nil? || input.empty?
25
+ MP4.logger.error 'usage: bin/divide_mp4_to_segments.rb <input.mp4> [output_dir]'
26
+ exit 1
27
+ end
28
+ unless File.file?(input)
29
+ MP4.logger.error "input not found: #{input}"
30
+ exit 1
31
+ end
32
+
33
+ output = ARGV[1]
34
+ output = ENV['MP4_OUTPUT'] if output.nil? || output.empty?
35
+ if output.nil? || output.empty?
36
+ base = File.basename(input, File.extname(input))
37
+ output = "out/fmp4/#{base}/"
38
+ end
39
+ FileUtils.mkdir_p(output)
40
+
41
+ MP4.logger.info "Input: #{input}"
42
+ MP4.logger.info "Output: #{output}"
43
+
44
+ mp4 = MP4.new(path: input)
45
+ mp4.save_fmp4_segments_to(output)
46
+
47
+ if ENV['XLSX']
48
+ xlsx_path = "out/xlsx/#{File.basename(input, File.extname(input))}.xlsx"
49
+ FileUtils.mkdir_p(File.dirname(xlsx_path))
50
+ mp4.build_xslx(xlsx_path)
51
+ MP4.logger.info "XLSX: #{xlsx_path}"
52
+ end
@@ -0,0 +1,11 @@
1
+ module M4S
2
+ module Binary
3
+ class Ftyp < BinData::Record
4
+ endian :big
5
+
6
+ string :major_brand, length: 4
7
+ uint32 :minor_version
8
+ rest :compatible_brands
9
+ end
10
+ end
11
+ end
File without changes
@@ -0,0 +1,18 @@
1
+ # mfhd = MP4::Binary::Mfhd.new(
2
+ # size: 16,
3
+ # version: 0,
4
+ # flags: 0,
5
+ # sequence_number: 1
6
+ # )
7
+ #
8
+ module M4S
9
+ module Binary
10
+ class Mfhd < BinData::Record
11
+ endian :big
12
+
13
+ uint8 :version
14
+ bit24 :flags
15
+ uint32 :sequence_number
16
+ end
17
+ end
18
+ end
@@ -0,0 +1,39 @@
1
+ module M4S::Binary
2
+ class Mvhd < BinData::Record
3
+ endian :big
4
+
5
+ bit1 :version, value: 0
6
+ array :flags, initial_length: 3, type: :uint8
7
+
8
+ uint32 :creation_time
9
+ uint32 :modification_time
10
+ uint32 :timescale
11
+ uint32 :duration
12
+
13
+ fixed_point16 :rate
14
+ fixed_point8 :volume
15
+
16
+ skip length: 2 + 4 + 4
17
+
18
+ array :matrix, initial_length: 9, type: :uint32
19
+
20
+ skip length: 4 * 6
21
+
22
+ uint32 :next_track_id
23
+ end
24
+ end
25
+ # ISO/IEC 14496-12:2015(E)
26
+ # SEMANTICS:
27
+ # `version` - is an integer that specifies the version of this box (0 or 1 in this specification) creation_time is an integer that declares the creation time of the presentation (in seconds
28
+ # `since` - midnight, Jan. 1, 1904, in UTC time)
29
+ # `modification_time` - is an integer that declares the most recent time the presentation was
30
+ # `modified` - (in seconds since midnight, Jan. 1, 1904, in UTC time)
31
+ # `timescale` - is an integer that specifies the time‐scale for the entire presentation; this is the
32
+ # `number` - of time units that pass in one second. For example, a time coordinate system that
33
+ # `measures` - time in sixtieths of a second has a time scale of 60.
34
+ # `duration` - is an integer that declares length of the presentation (in the indicated timescale). This
35
+ # `property` - is derived from the presentation’s tracks: the value of this field corresponds to the duration of the longest track in the presentation. If the duration cannot be determined then duration is set to all 1s.
36
+ # `rate` - is a fixed point 16.16 number that indicates the preferred rate to play the presentation; 1.0 (0x00010000) is normal forward playback
37
+ # `volume` - is a fixed point 8.8 number that indicates the preferred playback volume. 1.0 (0x0100) is full volume.
38
+ # `matrix - provides a transformation matrix for the video; (u,v,w) are restricted here to (0,0,1), hex values (0,0,0x40000000).
39
+ # `next_track_ID` - is a non‐zero integer that indicates a value to use for the track ID of the next track to be added to this presentation. Zero is not a valid track ID value. The value of next_track_ID shall be larger than the largest track‐ID in use. If this value is equal to all 1s (32‐bit maxint), and a new media track is to be added, then a search must be made in the file for an unused track identifier.
@@ -0,0 +1,38 @@
1
+ # sidx = MP4::Binary::Sidx.new(
2
+ # size: 52,
3
+ # version: 1,
4
+ # flags: 0,
5
+ # reference_id: 1,
6
+ # timescale: 24000,
7
+ # earliest_presentation_time: 0,
8
+ # first_offset: 0,
9
+ # references: [
10
+ # { reference_type: 0, size: 3040256, duration: 216216, starts_with_sap: 1, sap_type: 0, sap_delta_time: 0 }
11
+ # ]
12
+ # )
13
+
14
+ module M4S
15
+ module Binary
16
+ class Sidx < BinData::Record
17
+ endian :big
18
+
19
+ uint8 :version
20
+ bit24 :flags
21
+ uint32 :reference_id
22
+ uint32 :timescale
23
+ uint64 :earliest_presentation_time
24
+ uint64 :first_offset
25
+ uint16 :reserved, value: 0
26
+ uint16 :reference_count
27
+
28
+ array :references, initial_length: :reference_count do
29
+ bit1 :reference_type
30
+ bit31 :referenced_size
31
+ uint32 :subsegment_duration
32
+ bit1 :starts_with_sap
33
+ bit3 :sap_type
34
+ bit28 :sap_delta_time
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,17 @@
1
+ # styp = M4S::Binary::Styp.new(
2
+ # major_brand: "msdh",
3
+ # minor_version: 0,
4
+ # compatible_brands: ["msdh", "msix"]
5
+ # )
6
+
7
+ module M4S
8
+ module Binary
9
+ class Styp < BinData::Record
10
+ endian :big
11
+
12
+ string :major_brand, length: 4
13
+ uint32 :minor_version
14
+ array :compatible_brands, type: :string
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,25 @@
1
+ # tfdt = M4S::Binary::Tfdt.new(
2
+ # version: M4S::Binary::Tfdt::VERSION_1,
3
+ # flags: 0,
4
+ # base_media_decode_time: 1234567890
5
+ # )
6
+
7
+ # Track Fragment Base Media Decode Time Box
8
+ module M4S
9
+ module Binary
10
+ class Tfdt < BinData::Record
11
+ endian :big
12
+
13
+ VERSION_0 = 0
14
+ VERSION_1 = 1
15
+
16
+ uint8 :version
17
+ bit24 :flags
18
+
19
+ choice :base_media_decode_time, selection: :version do
20
+ uint32 VERSION_0
21
+ uint64 VERSION_1
22
+ end
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,38 @@
1
+ # tfhd = M4S::Binary::Tfhd.new(
2
+ # version: 0,
3
+ # flags: M4S::Binary::Tfhd::FLAGS.values.reduce(:|), # Пример: все опциональные поля присутствуют
4
+ # track_id: 1,
5
+ # base_data_offset: 0, # только если флаг 0x000001 установлен
6
+ # sample_description_index: 1, # только если флаг 0x000002 установлен
7
+ # default_sample_duration: 1001, # только если флаг 0x000008 установлен
8
+ # default_sample_size: 907, # только если флаг 0x000010 установлен
9
+ # default_sample_flags: 0 # только если флаг 0x000020 установлен
10
+ # )
11
+
12
+ # Track Fragment Header Box
13
+ module M4S
14
+ module Binary
15
+ class Tfhd < BinData::Record
16
+ endian :big
17
+
18
+ FLAGS = {
19
+ base_data_offset: 0x000001,
20
+ sample_description_index: 0x000002,
21
+ default_sample_duration: 0x000008,
22
+ default_sample_size: 0x000010,
23
+ default_sample_flags: 0x000020,
24
+ }.freeze
25
+
26
+ uint8 :version
27
+ bit24 :flags
28
+ uint32 :track_id
29
+
30
+ # Опциональные поля, которые зависят от флагов
31
+ uint64 :base_data_offset, onlyif: -> { flags & FLAGS[:base_data_offset] != 0 }
32
+ uint32 :sample_description_index, onlyif: -> { flags & FLAGS[:sample_description_index] != 0 }
33
+ uint32 :default_sample_duration, onlyif: -> { flags & FLAGS[:default_sample_duration] != 0 }
34
+ uint32 :default_sample_size, onlyif: -> { flags & FLAGS[:default_sample_size] != 0 }
35
+ uint32 :default_sample_flags, onlyif: -> { flags & FLAGS[:default_sample_flags] != 0 }
36
+ end
37
+ end
38
+ end