@torrent-tv/proxy 2.80.19 → 2.81.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/package.json +1 -1
- package/research/double-spawn-2026-09-10.md +171 -0
- package/services/disk/DiskSpace.js +150 -0
- package/services/disk/wire.js +60 -0
- package/services/encode/EncodeRun.js +25 -3
- package/services/encode/SegmentStore.js +137 -9
- package/services/hls-session-manager.js +26 -45
- package/services/orchestrators/EncodeOrchestrator.js +4 -1
- package/services/piece-store/allowance.js +107 -0
- package/services/piece-store/piece-disk-store.js +365 -0
- package/services/piece-store/shared-piece-store.js +1549 -1535
- package/services/torrent-worker/client.js +32 -0
- package/services/torrent-worker/pool-adapter.js +15 -0
- package/services/torrent-worker/protocol.js +9 -0
- package/services/torrent-worker/worker.js +8 -1
- package/services/viewer/positions.js +48 -0
- package/test/audio-inventory.test.js +176 -176
- package/test/auto-quality-step.test.js +514 -514
- package/test/concurrent-cost.test.js +138 -138
- package/test/coverage-follows-the-disk.test.js +191 -191
- package/test/coverage-map.test.js +195 -195
- package/test/declared-tracks.test.js +35 -35
- package/test/disk-space.test.js +150 -0
- package/test/encode-orchestrator.test.js +0 -3
- package/test/encode-run.test.js +5 -12
- package/test/held-request-width.test.js +155 -155
- package/test/helpers/encode-run.js +2 -2
- package/test/matroska-blocks.test.js +0 -0
- package/test/matroska-cues-track.test.js +192 -192
- package/test/mp4-composition-times.test.js +0 -0
- package/test/mp4-subtitles.test.js +173 -173
- package/test/one-authority.test.js +281 -220
- package/test/orchestrator-wired.test.js +199 -199
- package/test/packet-witness-ring.test.js +236 -236
- package/test/packet-witness.test.js +148 -148
- package/test/piece-disk-store.test.js +267 -0
- package/test/piece-reader.test.js +4 -4
- package/test/piece-store-eviction.test.js +17 -17
- package/test/piece-store-reservations.test.js +20 -1
- package/test/piece-store-slow-disk.test.js +16 -1
- package/test/read-window.test.js +6 -6
- package/test/run-intervals.test.js +100 -100
- package/test/seek-landing.test.js +109 -109
- package/test/segment-store-eviction.test.js +232 -0
- package/test/segments-are-shared.test.js +1 -1
- package/test/shared-piece-store.test.js +12 -12
- package/test/sidecar-naming.test.js +142 -142
- package/test/subtitle-cue-framing.test.js +200 -200
- package/test/subtitle-cue-walk.test.js +369 -369
- package/test/subtitle-defaults.test.js +97 -97
- package/test/subtitle-track-numbering.test.js +370 -370
- package/test/tail-duplication.test.js +167 -167
- package/test/tracks-begin-together.test.js +195 -195
- package/test/two-viewers-one-picture.test.js +374 -374
- package/test/video-facts.test.js +102 -102
- package/test/wedge-certainty.test.js +131 -131
- package/services/piece-store/disk-tier.js +0 -151
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,19 @@
|
|
|
1
|
+
## 2.81.1
|
|
2
|
+
|
|
3
|
+
- **Fix**: The `disk:` line is actually said. 2.81.0 built the reading and called it from nowhere, so the one thing that can answer "why is there no room" was absent from the log. The owner says it itself, once a pass, in the series beside the memory reading.
|
|
4
|
+
|
|
5
|
+
## 2.81.0
|
|
6
|
+
|
|
7
|
+
- **Fix**: EVERY ENCODER RAN TWICE, ON EVERY RUN, SINCE 2026-09-04. Building a run and starting it were two acts, so two owners each performed the second: the session manager built a run and started it, handed it back, and the orchestrator started it again. Field logs of 08-10 September: 207 runs against 414 spawns, no exception. Only the second process was reachable — this line overwrote the reference to the first — so a stop killed one and the other ran on, measured 105 seconds past its own run's death, with eleven processes writing at once on a four-core host whose budget said three. It doubled the processor, doubled the readers of the piece store (which is the pinned-piece deadlock's own trigger), and put two writers on one file name, defeating the rename 2.80.19 had just introduced: 1105 `could not publish … ENOENT` and a fatal `bufferAppendError` on both tracks at once under 2.80.19 itself.
|
|
8
|
+
**A run exists means its process is running.** `start()` is gone; the spawn is part of construction. There is no second act for a second owner to perform, and `test/one-authority.test.js` now asks what its own name promised — it counted where a run is BUILT and never where one is STARTED, which is why it passed throughout.
|
|
9
|
+
- **Fix**: The spilled pieces have a ceiling, and passing it gives disk back. `DiskTier` wrote every piece into one sparse file and answered `forget` by dropping a number from a set: the blocks stayed until the whole file went, and nothing this runtime offers punches a hole in a sparse file. Field 2026-08-31: a store holding 312-424 MB had written 14 400 MB, and the host's free space fell by every megabyte of it. `PieceDiskStore` gives each piece its own file, so removing one returns exactly its blocks; over its allowance it throws away the least recently used, and a piece being read is never the victim.
|
|
10
|
+
- **Fix**: One owner of the disk, read by everything that takes any of it. Three things wrote to the same disk and each read the free space as though it were alone — segments at a quarter of free plus a 2 GB floor, both chosen by hand; spilled pieces at nothing at all; diagnostics bounded by a count and never by a size. `services/disk/DiskSpace.js` reads it once and divides, by the rule memory already uses: what is free, plus what we hold, less what everything that is not us has been seen to need. `SEGMENT_STORE_FREE_SHARE`, `SEGMENT_STORE_FALLBACK_BYTES` and the local free-space read are deleted; the torrent thread's share travels the channel it already uses and the reply says what it holds.
|
|
11
|
+
- **Fix**: What goes first when the disk is short is decided by where the viewers are, not by when a directory was last read. Outputs nobody is watching, then what lies behind the earliest viewer furthest-behind-first, then what lies ahead of the furthest viewer. A segment a viewer is standing on is never taken. The idle rule stays beside it and answers the other question: material nobody needs should not sit on the owner's disk merely because there is room.
|
|
12
|
+
- **Fix**: A clean exit leaves nothing of ours. The root was removed only when it happened to be empty — from the first commit of this repository, never a decision — so a directory adopted at startup, owned by no session, survived the exit and was adopted again at the next start. That loop is what made an orphan permanent, and it is why 5.0 GB of segments from sessions that had ended hours before were on the addon host on 2026-09-10. Now everything the store owns goes, which also gives the startup sweep its meaning back: what is found then is from a kill.
|
|
13
|
+
- **Fix**: Every test spilled into the process working directory. They passed `spillDirectory` and the store reads `options.path`, so each wrote over the others' files.
|
|
14
|
+
- **Chore**: `allowance.js` holds the rule memory and disk now share, instead of one copy per resource. 991 checks pass, biome clean; the ceiling and the double spawn were each checked by breaking them on purpose — 6 of 9 and 1 of 13 go red.
|
|
15
|
+
- **Chore**: NOT yet seen in the field. What the next session must show: one ffmpeg per run in `run-state` (207 IDLE spawns and no STARTING spawns), no `could not publish … ENOENT`, a `disk:` line naming free space and each claimant's share, and `/tmp/torrent-tv-hls` empty after a clean stop.
|
|
16
|
+
|
|
1
17
|
## 2.80.19
|
|
2
18
|
|
|
3
19
|
- **Fix**: HALF A SEGMENT WAS SERVED TO THE PLAYER, AND THE SESSION NEVER RECOVERED. A piece was taken as finished when the NEXT number existed. That is sound for one writer walking forward and false the moment two runs share an output — which is what the plan gives an output whenever it places a second encoder, and what a one-segment interval guarantees. Field 2026-09-08: `segment-00057.mp4` was served at 2 268 361 bytes and then at 4 510 940, exactly half; `segment-00055.mp4` at 211 957 and then 2 620 617. The browser appended the half, refused the whole one for the rest of the session and repeated `bufferAppendError` to the end of the log with the picture frozen at 319.66 s.
|
package/package.json
CHANGED
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
# Каждый кодировщик запускается дважды, и второй процесс никто не убивает
|
|
2
|
+
|
|
3
|
+
Разбор восьми полевых сессий 8-10 сентября 2026: журнал прокси
|
|
4
|
+
(`/data/proxy.log` + `proxy.log.1`, 188 634 строки, версии 2.80.18 и 2.80.19) и
|
|
5
|
+
журнал браузера с дроплета (80 055 строк, восемь подписей сессий). Каждое
|
|
6
|
+
утверждение помечено тем, откуда взято: ЗАМЕРЕНО (число из журнала), ПРОЧИТАНО
|
|
7
|
+
(из исходников), ВЫВЕДЕНО (следствие двух первых).
|
|
8
|
+
|
|
9
|
+
## 1. Причина: две власти вызывают `start()` на одном прогоне
|
|
10
|
+
|
|
11
|
+
ПРОЧИТАНО. Прогон строится и ЗАПУСКАЕТСЯ в одном вызове:
|
|
12
|
+
`EncodeOrchestrator.#placeAt` → `makeRun` → `hls-session-manager.#makeRunAt`
|
|
13
|
+
(:3782) → `#startEncodeRun` (:5361), и последней строкой этой функции стоит
|
|
14
|
+
`run.start(because)` (:5546). Вернувшись, оркестратор вызывает
|
|
15
|
+
`run.start(because)` ещё раз (`EncodeOrchestrator.js:576`).
|
|
16
|
+
|
|
17
|
+
`EncodeRun.start()` (`EncodeRun.js:360`) защиты от повторного вызова не имеет:
|
|
18
|
+
строит аргументы заново, порождает второй процесс и ПЕРЕЗАПИСЫВАЕТ
|
|
19
|
+
`this.#process`. Обработчики первого процесса остаются подписанными на тот же
|
|
20
|
+
объект прогона.
|
|
21
|
+
|
|
22
|
+
Обе строки появились 2026-09-04, в двух соседних коммитах: `a09b278` («A run is
|
|
23
|
+
an object») поставила вызов в менеджере, `53785f2` («Encoders become classes») —
|
|
24
|
+
в оркестраторе.
|
|
25
|
+
|
|
26
|
+
ЗАМЕРЕНО, по переходам состояния за оба журнала:
|
|
27
|
+
|
|
28
|
+
```
|
|
29
|
+
207 IDLE --SPAWNED--> STARTING
|
|
30
|
+
207 STARTING --SPAWNED--> STARTING
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Ровно поровну. Двойной запуск происходит на КАЖДОМ прогоне без исключения: 207
|
|
34
|
+
прогонов, 414 процессов ffmpeg.
|
|
35
|
+
|
|
36
|
+
Ни один тест этого не ловит: в `test/encode-orchestrator.test.js` и соседних
|
|
37
|
+
`makeRun` — заглушка, которая прогон не запускает, поэтому вызов в менеджере не
|
|
38
|
+
исполняется ни разу.
|
|
39
|
+
|
|
40
|
+
## 2. Остановка убивает только второй процесс
|
|
41
|
+
|
|
42
|
+
ПРОЧИТАНО. `EncodeRun.stop()` шлёт SIGTERM в `this.#process` — то есть во
|
|
43
|
+
второй. Первый не адресуем ниоткуда: ссылки на него не осталось.
|
|
44
|
+
|
|
45
|
+
ЗАМЕРЕНО. Прогон `#453..#562` выхода `…grid=kf@38:video-only:v=38/copy`
|
|
46
|
+
остановлен в 09:12:01.178; его ffmpeg писал в поток ошибок до **09:13:46.938 —
|
|
47
|
+
105 секунд спустя**. Прогон `#463..#562` остановлен 09:12:10.204, последняя
|
|
48
|
+
строка 09:13:50.038 — 100 секунд спустя. Осиротевший процесс живёт до конца
|
|
49
|
+
своего отрезка, продолжая декодировать, читать из роя и писать файлы.
|
|
50
|
+
|
|
51
|
+
ЗАМЕРЕНО. Число РАЗНЫХ отрезков, писавших в поток ошибок в течение одной
|
|
52
|
+
секунды, на ОДНОМ выходе: 11 (09:13:31), 10 (09:13:32), 10 (09:13:34), 10
|
|
53
|
+
(09:13:05). Машина — CM4, четыре ядра. Бюджет в это время печатал `maxRuns=3`, и
|
|
54
|
+
машина состояний насчитывала не более трёх живых прогонов: **восемь из
|
|
55
|
+
одиннадцати процессов не видел никто.**
|
|
56
|
+
|
|
57
|
+
## 3. Что это ломает
|
|
58
|
+
|
|
59
|
+
### 3.1. Доказательство завершённости куска обойдено
|
|
60
|
+
|
|
61
|
+
Два процесса одного прогона получают ОДИН И ТОТ ЖЕ аргумент вывода —
|
|
62
|
+
`making-<метка>-%05d.mp4`, где метка равна первому номеру отрезка. Метка задумана
|
|
63
|
+
как имя прогона («отрезки не пересекаются, значит два живых прогона одного выхода
|
|
64
|
+
начинаются с разных номеров по построению»). Для двух процессов ОДНОГО прогона
|
|
65
|
+
это неверно: метка совпадает, и они пишут одно имя.
|
|
66
|
+
|
|
67
|
+
ЗАМЕРЕНО: **1105 строк** `segment store: could not publish making-…: ENOENT` —
|
|
68
|
+
один из двух переименовал файл, второму переименовывать нечего. По меткам: 904 на
|
|
69
|
+
`making-0`, остальные разбросаны по всем выходам и всем сессиям.
|
|
70
|
+
|
|
71
|
+
ВЫВЕДЕНО: это ровно та порча, ради которой 2.80.19 вводил рабочее имя — два
|
|
72
|
+
писателя на одно имя. Правка не отменена, она обойдена изнутри.
|
|
73
|
+
|
|
74
|
+
ЗАМЕРЕНО, что порча жива после 2.80.19: сессия `8cddc6f8` (10.09, 09:15:04) —
|
|
75
|
+
`fatal: bufferAppendError` одновременно на звуке (`segment-00072.mp4`) и на
|
|
76
|
+
картинке (`segment-00071.mp4`), `currentTime=712.61`. Сессия кончилась.
|
|
77
|
+
|
|
78
|
+
### 3.2. Механизм имени сам по себе исправен
|
|
79
|
+
|
|
80
|
+
ЗАМЕРЕНО на хосте дополнения: ffmpeg с `-f segment -segment_list pipe:3
|
|
81
|
+
-segment_list_flags +live` и шаблоном `making-0-%05d.mp4` пишет в четвёртый канал
|
|
82
|
+
имена `making-0-00000.mp4`, `…-00001`, `…-00002`, и файлы под этими именами лежат
|
|
83
|
+
на диске. ENOENT возникает не из-за имени и не из-за канала.
|
|
84
|
+
|
|
85
|
+
### 3.3. Двойное чтение из роя
|
|
86
|
+
|
|
87
|
+
ВЫВЕДЕНО. Каждый процесс открывает свой `/stream` (свой `createFragmentReader`),
|
|
88
|
+
поэтому читателей хранилища кусков вдвое больше, чем прогонов. Это спусковой
|
|
89
|
+
крючок тупика с закреплёнными кусками, где 2026-09-07 торрент был уничтожен при
|
|
90
|
+
четырёх читателях.
|
|
91
|
+
|
|
92
|
+
### 3.4. Процессорное время и задержки
|
|
93
|
+
|
|
94
|
+
ЗАМЕРЕНО. Ответы на опрос прогресса (тело около 500 байт): медиана 22 мс, p90
|
|
95
|
+
**2229 мс**, максимум **18 852 мс**; 362 ответа из 3516 дольше двух секунд.
|
|
96
|
+
ЗАМЕРЕНО. Предложение качества в те же минуты: `not offering 540p=0.10x
|
|
97
|
+
480p=0.11x 360p=0.12x 240p=0.13x`.
|
|
98
|
+
|
|
99
|
+
## 4. Что найдено рядом и причиной не является
|
|
100
|
+
|
|
101
|
+
### 4.1. Зритель простоял 44 минуты, и ни один прибор этого не сосчитал
|
|
102
|
+
|
|
103
|
+
Сессия `39c188b5` (08.09, 20:38-21:28). Зритель перемотал на 551.1 с, затем на
|
|
104
|
+
579.4 с. Картинка встала на 596.8 с, `readyState=1`, и до конца сессии не
|
|
105
|
+
сдвинулась: 528 одинаковых строк `decode t=596.8s frames=11787`.
|
|
106
|
+
|
|
107
|
+
ЗАМЕРЕНО: буфер держал `video=[…413..549.091] [859.192..886.136]` — перемотка
|
|
108
|
+
пришлась в дыру между 549.1 и 859.2. Браузер запросил за это время 283 куска, но
|
|
109
|
+
класть их в 596.8 было нечего.
|
|
110
|
+
|
|
111
|
+
ЗАМЕРЕНО: счётчик «сколько стояла картинка» не напечатал НИ ОДНОЙ строки за эту
|
|
112
|
+
сессию, а переход в состояние ожидания случился один раз. Счётчик закрывается
|
|
113
|
+
только по возобновлению, поэтому перерыв, который не кончился, не записывается
|
|
114
|
+
вовсе — худший случай невидим по построению.
|
|
115
|
+
|
|
116
|
+
ЗАМЕРЕНО: 42 `levelLoadError` за 48 минут при том, что прокси в те же секунды
|
|
117
|
+
отдавал `index.m3u8` (17 539 байт, `fetchMs=8-10`). Обе стороны о запросе
|
|
118
|
+
плейлиста говорят разное.
|
|
119
|
+
|
|
120
|
+
ЗАМЕРЕНО: браузер запрашивал `/v/540/index.m3u8` и `/v/480/index.m3u8`, декодируя
|
|
121
|
+
1920x1080 — hls.js по-прежнему сам ходит по ступеням при ошибках, вопреки
|
|
122
|
+
закреплению.
|
|
123
|
+
|
|
124
|
+
### 4.2. Куча главного потока выросла до 845 МБ за 26 секунд
|
|
125
|
+
|
|
126
|
+
ЗАМЕРЕНО (10.09): 09:15:20 heap=101 МБ, 09:15:31 — 378 МБ, 09:15:35 — 589 МБ,
|
|
127
|
+
09:15:46 — **813 МБ**, `rss=1940MB`. Написаны два снимка кучи процесса —
|
|
128
|
+
`heap-process-…-451543040` (431 МБ, 09:09) и `heap-process-…-885628928` (845 МБ,
|
|
129
|
+
09:15:44), оба в `/data`. **Их никто не открывал.** Это то самое чтение, которого
|
|
130
|
+
ждёт работа о необъяснённом росте памяти.
|
|
131
|
+
|
|
132
|
+
ЗАМЕРЕНО: после конца сессии куча падает до 35 МБ, а `rss` остаётся 979-981 МБ
|
|
133
|
+
четыре часа подряд без единого зрителя.
|
|
134
|
+
|
|
135
|
+
### 4.3. Ошибки разбора потока
|
|
136
|
+
|
|
137
|
+
ЗАМЕРЕНО: 20 294 строки `missing picture in access unit` и 20 118 строк `Invalid
|
|
138
|
+
NAL unit size` — 89 % журнала. Все на ветке КОПИРОВАНИЯ картинки файлов `.mp4`
|
|
139
|
+
(`grid=kf@38` — 629 строк с адресом, `kf@33` — 185, `kf@46` — 94). Причина не
|
|
140
|
+
установлена. Проверено, что общего состояния чтения у двух процессов нет: каждый
|
|
141
|
+
запрос `/stream` создаёт свой читатель.
|
|
142
|
+
|
|
143
|
+
### 4.4. Прогоны, не сделавшие ничего
|
|
144
|
+
|
|
145
|
+
ЗАМЕРЕНО: 153 прогона остановлены планом, из них **68 не произвели ни одного
|
|
146
|
+
куска**. Медиана жизни остановленного прогона 9636 мс, p90 26 719 мс, суммарно
|
|
147
|
+
1885 секунд. Причины остановки: 83 — «standing at #N scores worse than standing
|
|
148
|
+
at #M», 49 — «the film is no worse off without it». Довели свой отрезок до конца
|
|
149
|
+
48 из 207.
|
|
150
|
+
|
|
151
|
+
### 4.5. Мелочи, каждая с местом
|
|
152
|
+
|
|
153
|
+
1. Поле `live=` в строке плана печатается пустым 89 раз из 211 (`maxRuns=3 live=`).
|
|
154
|
+
2. `/tmp/torrent-tv-hls` — 5,0 ГБ готовых кусков от сессий, кончившихся часами
|
|
155
|
+
раньше; 3272 файла в десяти каталогах. Ничего не подметает.
|
|
156
|
+
3. `/data` — 3,3 ГБ, из них слепок памяти на 4,4 ГБ от 07.09 и 365 МБ снимков
|
|
157
|
+
кучи.
|
|
158
|
+
4. Предложение качества в первую секунду сессии переворачивается: 19:39:40.568
|
|
159
|
+
`not offering 1080p=0.86x (offering 720p 540p 480p 360p 240p)`, 19:39:41.058
|
|
160
|
+
`not offering 720p=0.24x … (offering 1080p)`. Первое читает копируемую картинку
|
|
161
|
+
как перекодирование.
|
|
162
|
+
|
|
163
|
+
## 5. Чего этот разбор НЕ устанавливает
|
|
164
|
+
|
|
165
|
+
1. почему ffmpeg на ветке копирования выдаёт ошибки разбора — связь с двойным
|
|
166
|
+
запуском не показана;
|
|
167
|
+
2. что именно держит 945 МБ резидентной памяти при куче в 35 МБ — снимки есть,
|
|
168
|
+
они не прочитаны;
|
|
169
|
+
3. почему браузер сообщает `levelLoadError`, когда прокси отдал плейлист без
|
|
170
|
+
ошибки;
|
|
171
|
+
4. откуда в буфере взялся оторванный остров 859..886 с.
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file One owner of the disk, read by everything that takes any of it.
|
|
3
|
+
*
|
|
4
|
+
* Three things on this proxy write to the same disk and, until now, each
|
|
5
|
+
* decided for itself how much it could take:
|
|
6
|
+
*
|
|
7
|
+
* - the segments an encoder produces, bounded by a quarter of what was free
|
|
8
|
+
* plus a floor of 2 GB, both numbers chosen out of nothing;
|
|
9
|
+
* - the pieces the memory store spills, bounded by nothing at all — 14 400 MB
|
|
10
|
+
* written in one fifty-minute viewing, field 2026-08-31;
|
|
11
|
+
* - the diagnostics we keep on purpose (core dumps, heap snapshots, packet
|
|
12
|
+
* captures), each bounded by a COUNT and none by a size: on the addon host
|
|
13
|
+
* two dumps and five snapshots came to 3.2 GB.
|
|
14
|
+
*
|
|
15
|
+
* Each read the free space as though it were the only claimant, so three
|
|
16
|
+
* ceilings each stood for the whole disk. This is the one place that reads it,
|
|
17
|
+
* and what it hands out is a share.
|
|
18
|
+
*
|
|
19
|
+
* THE RULE IS THE ONE MEMORY ALREADY USES, on the other reading: what is free
|
|
20
|
+
* now, plus what we already hold, less what everything that is not us has been
|
|
21
|
+
* seen to need. Nothing is a fraction chosen by hand.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import { OtherDemand, divideAllowance } from "../piece-store/allowance.js";
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* A consumer of the disk.
|
|
28
|
+
*
|
|
29
|
+
* @typedef {object} DiskConsumer
|
|
30
|
+
* @property {string} name - What it is called in the reading.
|
|
31
|
+
* @property {() => number} held - What it holds right now, in bytes.
|
|
32
|
+
* @property {() => number} wanted - What it would take if it could. A consumer
|
|
33
|
+
* that cannot say asks for what it holds, which is the honest statement of a
|
|
34
|
+
* thing that only grows when something arrives.
|
|
35
|
+
* @property {(allowanceBytes: number) => void} allow - Told its share.
|
|
36
|
+
*/
|
|
37
|
+
|
|
38
|
+
export class DiskSpace {
|
|
39
|
+
/** @type {Map<string, DiskConsumer>} */
|
|
40
|
+
#consumers = new Map();
|
|
41
|
+
|
|
42
|
+
#otherDemand = new OtherDemand();
|
|
43
|
+
|
|
44
|
+
#readFree;
|
|
45
|
+
|
|
46
|
+
#logger;
|
|
47
|
+
|
|
48
|
+
/** The last division, for the reading. @type {{ name: string, held: number, allowed: number }[]} */
|
|
49
|
+
#last = [];
|
|
50
|
+
|
|
51
|
+
#freeBytes = 0;
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* @param {object} params
|
|
55
|
+
* @param {() => Promise<number | null>} params.readFree - What the machine
|
|
56
|
+
* says is free on the disk these consumers share.
|
|
57
|
+
* @param {{ info: (line: string) => void, warn?: (line: string) => void }} [params.logger]
|
|
58
|
+
*/
|
|
59
|
+
constructor({ readFree, logger = null }) {
|
|
60
|
+
this.#readFree = readFree;
|
|
61
|
+
this.#logger = logger;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Register a consumer. Registering twice under one name replaces it.
|
|
66
|
+
*
|
|
67
|
+
* @param {DiskConsumer} consumer
|
|
68
|
+
* @returns {void}
|
|
69
|
+
*/
|
|
70
|
+
register(consumer) {
|
|
71
|
+
this.#consumers.set(consumer.name, consumer);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* @param {string} name
|
|
76
|
+
* @returns {void}
|
|
77
|
+
*/
|
|
78
|
+
forget(name) {
|
|
79
|
+
this.#consumers.delete(name);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Read the disk, divide it, and tell each consumer its share.
|
|
84
|
+
*
|
|
85
|
+
* Called on the same timer that revises memory: a disk that fills while a
|
|
86
|
+
* film is playing must lower the ceilings, not keep ones taken when it was
|
|
87
|
+
* empty. That is the mistake memory made until 2026-08-28, and every disk
|
|
88
|
+
* ceiling on this proxy made until now.
|
|
89
|
+
*
|
|
90
|
+
* @returns {Promise<{ freeBytes: number, allowanceBytes: number, shares: { name: string, held: number, allowed: number }[] }>}
|
|
91
|
+
*/
|
|
92
|
+
async revise() {
|
|
93
|
+
const consumers = [...this.#consumers.values()];
|
|
94
|
+
if (consumers.length === 0) {
|
|
95
|
+
return { freeBytes: 0, allowanceBytes: 0, shares: [] };
|
|
96
|
+
}
|
|
97
|
+
const held = consumers.reduce((sum, consumer) => sum + Math.max(0, consumer.held()), 0);
|
|
98
|
+
const free = await this.#readFree();
|
|
99
|
+
// WITHOUT A READING, NOTHING IS ALLOWED TO GROW. A disk whose free space
|
|
100
|
+
// cannot be read is not a disk with room; answering "unbounded" there is how
|
|
101
|
+
// the spill file came to have no limit in the first place.
|
|
102
|
+
this.#freeBytes = Number.isFinite(free) && free !== null ? Math.max(0, free) : 0;
|
|
103
|
+
const reserve = this.#otherDemand.note(this.#freeBytes, held);
|
|
104
|
+
const allowance = Math.max(0, this.#freeBytes + held - reserve);
|
|
105
|
+
const shares = divideAllowance(
|
|
106
|
+
consumers.map((consumer) => Math.max(0, consumer.wanted())),
|
|
107
|
+
allowance
|
|
108
|
+
);
|
|
109
|
+
this.#last = consumers.map((consumer, position) => ({
|
|
110
|
+
name: consumer.name,
|
|
111
|
+
held: consumer.held(),
|
|
112
|
+
allowed: shares[position]
|
|
113
|
+
}));
|
|
114
|
+
for (const [position, consumer] of consumers.entries()) {
|
|
115
|
+
consumer.allow(shares[position]);
|
|
116
|
+
}
|
|
117
|
+
// SAID, every pass, in the series beside the memory reading. "Why is there
|
|
118
|
+
// no room" was a question no log could answer: the ceilings were worked out
|
|
119
|
+
// in three places and not one of them was printed beside the others.
|
|
120
|
+
this.#logger?.info?.(this.describe());
|
|
121
|
+
return { freeBytes: this.#freeBytes, allowanceBytes: allowance, shares: this.#last };
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* One line: what the disk has, what each consumer holds, and what it may.
|
|
126
|
+
*
|
|
127
|
+
* Said because "why is there no room" is otherwise a question no log can
|
|
128
|
+
* answer — the three ceilings were computed in three places and none of them
|
|
129
|
+
* was printed beside the others.
|
|
130
|
+
*
|
|
131
|
+
* @returns {string}
|
|
132
|
+
*/
|
|
133
|
+
describe() {
|
|
134
|
+
if (this.#last.length === 0) {
|
|
135
|
+
return "disk: nothing has claimed any yet";
|
|
136
|
+
}
|
|
137
|
+
const parts = this.#last.map(
|
|
138
|
+
(share) => `${share.name} ${megabytes(share.held)} of ${megabytes(share.allowed)}`
|
|
139
|
+
);
|
|
140
|
+
return `disk: ${megabytes(this.#freeBytes)} free; ${parts.join(", ")}`;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* @param {number} bytes
|
|
146
|
+
* @returns {string}
|
|
147
|
+
*/
|
|
148
|
+
function megabytes(bytes) {
|
|
149
|
+
return `${Math.round(Math.max(0, bytes) / (1024 * 1024))}MB`;
|
|
150
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file Who takes disk, and how each of them is told its share.
|
|
3
|
+
*
|
|
4
|
+
* Kept apart from the owner because the owner knows nothing about this proxy —
|
|
5
|
+
* it reads a number, divides it and hands out shares — and kept out of the
|
|
6
|
+
* session manager because a list of claimants is not a fact about a session.
|
|
7
|
+
*
|
|
8
|
+
* Two claimants today. The diagnostics are a third and are not here yet: they
|
|
9
|
+
* are bounded by a COUNT and never by a size, which on the addon host let two
|
|
10
|
+
* core dumps and five heap snapshots come to 3.2 GB, and they cannot simply be
|
|
11
|
+
* thrown away when space is short — a dump is the only evidence of the death it
|
|
12
|
+
* records. That needs a rule of its own.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { DiskSpace } from "./DiskSpace.js";
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Build the owner of the disk and register everything that takes any of it.
|
|
19
|
+
*
|
|
20
|
+
* @param {object} params
|
|
21
|
+
* @param {{ root: string, stats: () => { bytes: number } }} params.segmentStore
|
|
22
|
+
* @param {{ spilledBytes?: number, allowSpillBytes?: (bytes: number) => unknown }} [params.torrentPool]
|
|
23
|
+
* @param {(directory: string) => Promise<number | null>} params.readFree
|
|
24
|
+
* @param {{ info: Function, warn?: Function }} [params.logger]
|
|
25
|
+
* @returns {{ revise: () => Promise<unknown>, segmentBytes: () => number, describe: () => string }}
|
|
26
|
+
* What the segments may hold is asked for rather than pushed: zero until the
|
|
27
|
+
* first revision, and zero stops growth rather than licensing it.
|
|
28
|
+
*/
|
|
29
|
+
export function wireDiskSpace({ segmentStore, torrentPool, readFree, logger }) {
|
|
30
|
+
const space = new DiskSpace({ readFree: () => readFree(segmentStore.root), logger });
|
|
31
|
+
let segmentBytes = 0;
|
|
32
|
+
space.register({
|
|
33
|
+
name: "segments",
|
|
34
|
+
held: () => segmentStore.stats().bytes,
|
|
35
|
+
// The whole of every film anybody is watching. There is no smaller honest
|
|
36
|
+
// answer, so it asks for everything and is cut in proportion like the rest.
|
|
37
|
+
wanted: () => Number.MAX_SAFE_INTEGER,
|
|
38
|
+
allow: (bytes) => {
|
|
39
|
+
segmentBytes = bytes;
|
|
40
|
+
}
|
|
41
|
+
});
|
|
42
|
+
if (typeof torrentPool?.allowSpillBytes === "function") {
|
|
43
|
+
// The pieces the memory store spills. They live on the torrent thread, so
|
|
44
|
+
// the share travels the channel that already carries everything else, and
|
|
45
|
+
// the reply says what they hold — one exchange, both directions.
|
|
46
|
+
space.register({
|
|
47
|
+
name: "spilled pieces",
|
|
48
|
+
held: () => torrentPool.spilledBytes ?? 0,
|
|
49
|
+
wanted: () => Number.MAX_SAFE_INTEGER,
|
|
50
|
+
allow: (bytes) => {
|
|
51
|
+
void torrentPool.allowSpillBytes?.(bytes);
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
return {
|
|
56
|
+
revise: () => space.revise(),
|
|
57
|
+
segmentBytes: () => segmentBytes,
|
|
58
|
+
describe: () => space.describe()
|
|
59
|
+
};
|
|
60
|
+
}
|
|
@@ -201,6 +201,9 @@ export class EncodeRun {
|
|
|
201
201
|
* kept so a failure can quote what produced it.
|
|
202
202
|
* @param {boolean} [params.usesExplicitCuts] - Whether this run cuts at times
|
|
203
203
|
* it was given, which decides how a segment is judged finished.
|
|
204
|
+
* @param {string} params.because - Why this encoder is being put on the
|
|
205
|
+
* machine, in words. Recorded with the argument list: a start whose cause is
|
|
206
|
+
* not written down cannot be told from any other when several runs exist.
|
|
204
207
|
* @param {(name: string) => number | null} [params.indexOfName] - The number
|
|
205
208
|
* a closed piece's name carries. How a piece is named belongs to the format
|
|
206
209
|
* that writes it, so it arrives as a plain function rather than this class
|
|
@@ -222,7 +225,8 @@ export class EncodeRun {
|
|
|
222
225
|
inputUnavailable,
|
|
223
226
|
argsDescribed = "",
|
|
224
227
|
usesExplicitCuts = false,
|
|
225
|
-
indexOfName
|
|
228
|
+
indexOfName,
|
|
229
|
+
because = "no reason was given"
|
|
226
230
|
}) {
|
|
227
231
|
this.address = address;
|
|
228
232
|
this.encoder = encoder;
|
|
@@ -244,6 +248,9 @@ export class EncodeRun {
|
|
|
244
248
|
this.indexOfName = typeof indexOfName === "function" ? indexOfName : () => null;
|
|
245
249
|
/** The last thing ffmpeg said on stderr, which is what a failure is explained by. */
|
|
246
250
|
this.lastError = "";
|
|
251
|
+
// EXISTING IS RUNNING. There is no moment at which a built run is not yet a
|
|
252
|
+
// process, so there is no second act for two owners to perform.
|
|
253
|
+
this.#begin(because);
|
|
247
254
|
}
|
|
248
255
|
|
|
249
256
|
/** @returns {string} */
|
|
@@ -348,7 +355,22 @@ export class EncodeRun {
|
|
|
348
355
|
}
|
|
349
356
|
|
|
350
357
|
/**
|
|
351
|
-
*
|
|
358
|
+
* Put the process on the machine, and say why.
|
|
359
|
+
*
|
|
360
|
+
* PRIVATE, AND CALLED ONCE, from the constructor. It was public until
|
|
361
|
+
* 2026-09-10, and two places called it: the session manager built a run and
|
|
362
|
+
* started it, then handed it back to the orchestrator, which started it
|
|
363
|
+
* again. Every run of every session therefore had TWO ffmpeg processes on one
|
|
364
|
+
* output writing one set of names — 207 runs against 414 spawns in the field
|
|
365
|
+
* logs of 08-10 September, without a single exception. Only the second was
|
|
366
|
+
* reachable afterwards, because this line overwrote the reference to the
|
|
367
|
+
* first, so `stop` killed one and the other ran on: measured 105 seconds past
|
|
368
|
+
* its own run's death, eleven processes writing at once on a four-core host
|
|
369
|
+
* whose budget said three.
|
|
370
|
+
*
|
|
371
|
+
* The guard against that is not a check but the absence of a second act: a
|
|
372
|
+
* run exists means its process is running, so there is nothing anybody can
|
|
373
|
+
* call twice.
|
|
352
374
|
*
|
|
353
375
|
* The reason is not decoration: a start whose cause is not recorded cannot be
|
|
354
376
|
* told from any other start when several runs exist at once, and the argument
|
|
@@ -357,7 +379,7 @@ export class EncodeRun {
|
|
|
357
379
|
*
|
|
358
380
|
* @param {string} because
|
|
359
381
|
*/
|
|
360
|
-
|
|
382
|
+
#begin(because) {
|
|
361
383
|
const args = this.buildArgs();
|
|
362
384
|
this.#startedAt = this.now();
|
|
363
385
|
this.logger.info(
|
|
@@ -481,6 +481,31 @@ export class SegmentStore {
|
|
|
481
481
|
this.#logger.info(`segment-store dropped ${directoryNameFor(key)} (${because})`);
|
|
482
482
|
}
|
|
483
483
|
|
|
484
|
+
/**
|
|
485
|
+
* Throw away everything this store owns, and the root with it.
|
|
486
|
+
*
|
|
487
|
+
* For a clean exit. What is left on disk afterwards is by definition from a
|
|
488
|
+
* kill, which is the case the startup sweep exists for — and without this the
|
|
489
|
+
* sweep adopts, the exit leaves, and the next start adopts again, for ever.
|
|
490
|
+
*
|
|
491
|
+
* @param {string} because
|
|
492
|
+
* @returns {number} How many outputs went.
|
|
493
|
+
*/
|
|
494
|
+
dropAll(because) {
|
|
495
|
+
let dropped = 0;
|
|
496
|
+
for (const key of [...this.#formats.keys()]) {
|
|
497
|
+
this.drop(key, because);
|
|
498
|
+
dropped += 1;
|
|
499
|
+
}
|
|
500
|
+
try {
|
|
501
|
+
rmSync(this.#root, { recursive: true, force: true });
|
|
502
|
+
} catch {
|
|
503
|
+
// Another process may share the root and hold a directory open. What is
|
|
504
|
+
// ours is gone either way.
|
|
505
|
+
}
|
|
506
|
+
return dropped;
|
|
507
|
+
}
|
|
508
|
+
|
|
484
509
|
/**
|
|
485
510
|
* Keep only what is still being read, and only as much of it as there is room
|
|
486
511
|
* for.
|
|
@@ -496,13 +521,34 @@ export class SegmentStore {
|
|
|
496
521
|
* that is the cap's — but to stop an output nobody has touched in hours from
|
|
497
522
|
* sitting there for the life of the process.
|
|
498
523
|
*
|
|
524
|
+
* TWO RULES, ANSWERING TWO QUESTIONS. Kept apart because they were briefly
|
|
525
|
+
* proposed as one and that was wrong: material nobody needs should not sit on
|
|
526
|
+
* the owner's disk merely because there is room for it, and material everyone
|
|
527
|
+
* needs must still go when there is no room. The first is time, the second is
|
|
528
|
+
* space.
|
|
529
|
+
*
|
|
530
|
+
* WHAT GOES FIRST WHEN THERE IS NO ROOM is decided by where the viewers are,
|
|
531
|
+
* not by when a directory was last read. Behind every viewer of an output is
|
|
532
|
+
* material that has been played and will not be asked for again unless
|
|
533
|
+
* somebody seeks back; ahead of the furthest viewer is material that will be
|
|
534
|
+
* asked for, eventually. So the order is: outputs nobody is watching at all,
|
|
535
|
+
* then what lies behind the earliest viewer, furthest behind first, then what
|
|
536
|
+
* lies ahead of the furthest viewer, furthest ahead first. It is the priority
|
|
537
|
+
* map's own order read from the other end.
|
|
538
|
+
*
|
|
539
|
+
* A segment a viewer is standing on is never a victim.
|
|
540
|
+
*
|
|
499
541
|
* @param {object} params
|
|
500
542
|
* @param {number} params.idleMs - Untouched for longer than this, and it goes.
|
|
501
|
-
* @param {number} params.maxBytes - The most the whole store may hold.
|
|
502
|
-
*
|
|
503
|
-
*
|
|
543
|
+
* @param {number} params.maxBytes - The most the whole store may hold.
|
|
544
|
+
* @param {(key: string) => number[]} [params.viewersAt] - Where the viewers of
|
|
545
|
+
* an output stand, as segment numbers. An empty answer means nobody is
|
|
546
|
+
* watching it, which is what makes its segments the first to go. Absent, the
|
|
547
|
+
* store has nothing to order by and falls back to the oldest directory —
|
|
548
|
+
* which is what it did before it could be told.
|
|
549
|
+
* @returns {{ droppedIdle: number, droppedForRoom: number, segmentsRemoved: number, bytes: number }}
|
|
504
550
|
*/
|
|
505
|
-
enforce({ idleMs, maxBytes }) {
|
|
551
|
+
enforce({ idleMs, maxBytes, viewersAt = null }) {
|
|
506
552
|
const now = this.#now();
|
|
507
553
|
let droppedIdle = 0;
|
|
508
554
|
for (const [key, touchedAt] of [...this.#touched]) {
|
|
@@ -511,11 +557,12 @@ export class SegmentStore {
|
|
|
511
557
|
droppedIdle += 1;
|
|
512
558
|
}
|
|
513
559
|
}
|
|
514
|
-
let droppedForRoom = 0;
|
|
515
560
|
let held = this.stats().bytes;
|
|
516
|
-
if (Number.isFinite(maxBytes)
|
|
517
|
-
|
|
518
|
-
|
|
561
|
+
if (!Number.isFinite(maxBytes) || maxBytes <= 0 || held <= maxBytes) {
|
|
562
|
+
return { droppedIdle, droppedForRoom: 0, segmentsRemoved: 0, bytes: held };
|
|
563
|
+
}
|
|
564
|
+
if (typeof viewersAt !== "function") {
|
|
565
|
+
let droppedForRoom = 0;
|
|
519
566
|
const byAge = [...this.#touched.entries()].sort((left, right) => left[1] - right[1]);
|
|
520
567
|
for (const [key] of byAge) {
|
|
521
568
|
if (held <= maxBytes) {
|
|
@@ -526,8 +573,81 @@ export class SegmentStore {
|
|
|
526
573
|
held -= size;
|
|
527
574
|
droppedForRoom += 1;
|
|
528
575
|
}
|
|
576
|
+
return { droppedIdle, droppedForRoom, segmentsRemoved: 0, bytes: held };
|
|
529
577
|
}
|
|
530
|
-
|
|
578
|
+
|
|
579
|
+
let segmentsRemoved = 0;
|
|
580
|
+
for (const victim of this.#leastWantedFirst(viewersAt)) {
|
|
581
|
+
if (held <= maxBytes) {
|
|
582
|
+
break;
|
|
583
|
+
}
|
|
584
|
+
held -= this.#removeSegment(victim.key, victim.index);
|
|
585
|
+
segmentsRemoved += 1;
|
|
586
|
+
}
|
|
587
|
+
if (segmentsRemoved > 0) {
|
|
588
|
+
this.#logger.info(
|
|
589
|
+
`segment-store removed ${segmentsRemoved} segment(s) for room: ` +
|
|
590
|
+
`${megabytes(held)} of ${megabytes(maxBytes)} allowed`
|
|
591
|
+
);
|
|
592
|
+
}
|
|
593
|
+
return { droppedIdle, droppedForRoom: 0, segmentsRemoved, bytes: held };
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
/**
|
|
597
|
+
* Every segment in the store, least wanted first.
|
|
598
|
+
*
|
|
599
|
+
* @param {(key: string) => number[]} viewersAt
|
|
600
|
+
* @returns {{ key: string, index: number }[]}
|
|
601
|
+
*/
|
|
602
|
+
#leastWantedFirst(viewersAt) {
|
|
603
|
+
const candidates = [];
|
|
604
|
+
for (const key of this.#formats.keys()) {
|
|
605
|
+
const positions = (viewersAt(key) ?? []).filter((at) => Number.isInteger(at));
|
|
606
|
+
const earliest = positions.length > 0 ? Math.min(...positions) : null;
|
|
607
|
+
const furthest = positions.length > 0 ? Math.max(...positions) : null;
|
|
608
|
+
for (const index of this.refresh(key).byNumber.keys()) {
|
|
609
|
+
if (earliest === null) {
|
|
610
|
+
// Nobody is watching this output at all. Everything it holds is worth
|
|
611
|
+
// less than anything somebody is on their way to.
|
|
612
|
+
candidates.push({ key, index, rank: 0, distance: index });
|
|
613
|
+
continue;
|
|
614
|
+
}
|
|
615
|
+
if (positions.includes(index)) {
|
|
616
|
+
continue;
|
|
617
|
+
}
|
|
618
|
+
if (index < earliest) {
|
|
619
|
+
candidates.push({ key, index, rank: 1, distance: earliest - index });
|
|
620
|
+
} else {
|
|
621
|
+
candidates.push({ key, index, rank: 2, distance: index - /** @type {number} */ (furthest) });
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
return candidates
|
|
626
|
+
.sort((left, right) => (left.rank !== right.rank ? left.rank - right.rank : right.distance - left.distance))
|
|
627
|
+
.map(({ key, index }) => ({ key, index }));
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
/**
|
|
631
|
+
* Take one segment off the disk.
|
|
632
|
+
*
|
|
633
|
+
* @param {string} key
|
|
634
|
+
* @param {number} index
|
|
635
|
+
* @returns {number} What it weighed.
|
|
636
|
+
*/
|
|
637
|
+
#removeSegment(key, index) {
|
|
638
|
+
const full = this.refresh(key).byNumber.get(index);
|
|
639
|
+
if (!full) {
|
|
640
|
+
return 0;
|
|
641
|
+
}
|
|
642
|
+
let size = 0;
|
|
643
|
+
try {
|
|
644
|
+
size = statSync(full, { throwIfNoEntry: false })?.size ?? 0;
|
|
645
|
+
rmSync(full, { force: true });
|
|
646
|
+
} catch {
|
|
647
|
+
// Gone already, or refused. The next refresh reports what is really there.
|
|
648
|
+
}
|
|
649
|
+
this.#held.delete(key);
|
|
650
|
+
return size;
|
|
531
651
|
}
|
|
532
652
|
|
|
533
653
|
/**
|
|
@@ -665,3 +785,11 @@ export class SegmentStore {
|
|
|
665
785
|
return { adopted, dropped, unprovenRemoved };
|
|
666
786
|
}
|
|
667
787
|
}
|
|
788
|
+
|
|
789
|
+
/**
|
|
790
|
+
* @param {number} bytes
|
|
791
|
+
* @returns {string}
|
|
792
|
+
*/
|
|
793
|
+
function megabytes(bytes) {
|
|
794
|
+
return `${Math.round(Math.max(0, bytes) / (1024 * 1024))}MB`;
|
|
795
|
+
}
|