@torrent-tv/proxy 2.80.18 → 2.81.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/docs/encode-architecture.md +51 -2
- package/package.json +1 -1
- package/research/double-spawn-2026-09-10.md +171 -0
- package/services/disk/DiskSpace.js +146 -0
- package/services/disk/wire.js +60 -0
- package/services/encode/EncodeRun.js +37 -9
- package/services/encode/SegmentStore.js +284 -232
- package/services/encode/run-command.js +16 -1
- package/services/hls-session-manager.js +31 -128
- package/services/orchestrators/EncodeOrchestrator.js +52 -38
- package/services/piece-store/allowance.js +107 -0
- package/services/piece-store/piece-disk-store.js +365 -0
- package/services/piece-store/shared-piece-store.js +1549 -1535
- package/services/segment-formats/fmp4.js +54 -0
- package/services/segment-formats/mpegts.js +54 -0
- package/services/torrent-worker/client.js +32 -0
- package/services/torrent-worker/pool-adapter.js +15 -0
- package/services/torrent-worker/protocol.js +9 -0
- package/services/torrent-worker/worker.js +8 -1
- package/services/viewer/positions.js +48 -0
- package/test/audio-inventory.test.js +176 -176
- package/test/auto-quality-step.test.js +514 -514
- package/test/concurrent-cost.test.js +138 -138
- package/test/coverage-follows-the-disk.test.js +191 -187
- package/test/coverage-map.test.js +195 -195
- package/test/declared-tracks.test.js +35 -35
- package/test/disk-space.test.js +138 -0
- package/test/encode-orchestrator.test.js +0 -3
- package/test/encode-run.test.js +5 -12
- package/test/held-request-width.test.js +155 -155
- package/test/helpers/encode-run.js +2 -2
- package/test/matroska-blocks.test.js +0 -0
- package/test/matroska-cues-track.test.js +192 -192
- package/test/mp4-composition-times.test.js +0 -0
- package/test/mp4-subtitles.test.js +173 -173
- package/test/one-authority.test.js +281 -220
- package/test/orchestrator-wired.test.js +199 -195
- package/test/packet-witness-ring.test.js +236 -236
- package/test/packet-witness.test.js +148 -148
- package/test/piece-disk-store.test.js +267 -0
- package/test/piece-reader.test.js +4 -4
- package/test/piece-store-eviction.test.js +17 -17
- package/test/piece-store-reservations.test.js +20 -1
- package/test/piece-store-slow-disk.test.js +16 -1
- package/test/produced-copy-choice.test.js +258 -358
- package/test/read-window.test.js +6 -6
- package/test/run-intervals.test.js +100 -100
- package/test/seek-landing.test.js +109 -109
- package/test/segment-serve-wiring.test.js +8 -9
- package/test/segment-store-eviction.test.js +232 -0
- package/test/segment-store.test.js +238 -216
- package/test/segments-are-shared.test.js +1 -1
- package/test/shared-piece-store.test.js +12 -12
- package/test/sidecar-naming.test.js +142 -142
- package/test/subtitle-cue-framing.test.js +200 -200
- package/test/subtitle-cue-walk.test.js +369 -369
- package/test/subtitle-defaults.test.js +97 -97
- package/test/subtitle-track-numbering.test.js +370 -370
- package/test/tail-duplication.test.js +167 -167
- package/test/tracks-begin-together.test.js +195 -195
- package/test/two-viewers-one-picture.test.js +374 -374
- package/test/video-facts.test.js +102 -102
- package/test/wedge-certainty.test.js +131 -131
- package/services/encode/open-piece.js +0 -135
- package/services/piece-store/disk-tier.js +0 -151
- package/test/open-piece.test.js +0 -152
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,25 @@
|
|
|
1
|
+
## 2.81.0
|
|
2
|
+
|
|
3
|
+
- **Fix**: EVERY ENCODER RAN TWICE, ON EVERY RUN, SINCE 2026-09-04. Building a run and starting it were two acts, so two owners each performed the second: the session manager built a run and started it, handed it back, and the orchestrator started it again. Field logs of 08-10 September: 207 runs against 414 spawns, no exception. Only the second process was reachable — this line overwrote the reference to the first — so a stop killed one and the other ran on, measured 105 seconds past its own run's death, with eleven processes writing at once on a four-core host whose budget said three. It doubled the processor, doubled the readers of the piece store (which is the pinned-piece deadlock's own trigger), and put two writers on one file name, defeating the rename 2.80.19 had just introduced: 1105 `could not publish … ENOENT` and a fatal `bufferAppendError` on both tracks at once under 2.80.19 itself.
|
|
4
|
+
**A run exists means its process is running.** `start()` is gone; the spawn is part of construction. There is no second act for a second owner to perform, and `test/one-authority.test.js` now asks what its own name promised — it counted where a run is BUILT and never where one is STARTED, which is why it passed throughout.
|
|
5
|
+
- **Fix**: The spilled pieces have a ceiling, and passing it gives disk back. `DiskTier` wrote every piece into one sparse file and answered `forget` by dropping a number from a set: the blocks stayed until the whole file went, and nothing this runtime offers punches a hole in a sparse file. Field 2026-08-31: a store holding 312-424 MB had written 14 400 MB, and the host's free space fell by every megabyte of it. `PieceDiskStore` gives each piece its own file, so removing one returns exactly its blocks; over its allowance it throws away the least recently used, and a piece being read is never the victim.
|
|
6
|
+
- **Fix**: One owner of the disk, read by everything that takes any of it. Three things wrote to the same disk and each read the free space as though it were alone — segments at a quarter of free plus a 2 GB floor, both chosen by hand; spilled pieces at nothing at all; diagnostics bounded by a count and never by a size. `services/disk/DiskSpace.js` reads it once and divides, by the rule memory already uses: what is free, plus what we hold, less what everything that is not us has been seen to need. `SEGMENT_STORE_FREE_SHARE`, `SEGMENT_STORE_FALLBACK_BYTES` and the local free-space read are deleted; the torrent thread's share travels the channel it already uses and the reply says what it holds.
|
|
7
|
+
- **Fix**: What goes first when the disk is short is decided by where the viewers are, not by when a directory was last read. Outputs nobody is watching, then what lies behind the earliest viewer furthest-behind-first, then what lies ahead of the furthest viewer. A segment a viewer is standing on is never taken. The idle rule stays beside it and answers the other question: material nobody needs should not sit on the owner's disk merely because there is room.
|
|
8
|
+
- **Fix**: A clean exit leaves nothing of ours. The root was removed only when it happened to be empty — from the first commit of this repository, never a decision — so a directory adopted at startup, owned by no session, survived the exit and was adopted again at the next start. That loop is what made an orphan permanent, and it is why 5.0 GB of segments from sessions that had ended hours before were on the addon host on 2026-09-10. Now everything the store owns goes, which also gives the startup sweep its meaning back: what is found then is from a kill.
|
|
9
|
+
- **Fix**: Every test spilled into the process working directory. They passed `spillDirectory` and the store reads `options.path`, so each wrote over the others' files.
|
|
10
|
+
- **Chore**: `allowance.js` holds the rule memory and disk now share, instead of one copy per resource. 991 checks pass, biome clean; the ceiling and the double spawn were each checked by breaking them on purpose — 6 of 9 and 1 of 13 go red.
|
|
11
|
+
- **Chore**: NOT yet seen in the field. What the next session must show: one ffmpeg per run in `run-state` (207 IDLE spawns and no STARTING spawns), no `could not publish … ENOENT`, a `disk:` line naming free space and each claimant's share, and `/tmp/torrent-tv-hls` empty after a clean stop.
|
|
12
|
+
|
|
13
|
+
## 2.80.19
|
|
14
|
+
|
|
15
|
+
- **Fix**: HALF A SEGMENT WAS SERVED TO THE PLAYER, AND THE SESSION NEVER RECOVERED. A piece was taken as finished when the NEXT number existed. That is sound for one writer walking forward and false the moment two runs share an output — which is what the plan gives an output whenever it places a second encoder, and what a one-segment interval guarantees. Field 2026-09-08: `segment-00057.mp4` was served at 2 268 361 bytes and then at 4 510 940, exactly half; `segment-00055.mp4` at 211 957 and then 2 620 617. The browser appended the half, refused the whole one for the rest of the session and repeated `bufferAppendError` to the end of the log with the picture frozen at 319.66 s.
|
|
16
|
+
**A piece's NAME is the proof now, and there is no second one.** It is written as `making-<from>-00057.mp4` — tagged with the first number of its run's stretch — and takes `segment-00057.mp4` when the encoder says on its own channel that it has closed it, which is one rename inside one directory. The `hls` branch needed nothing added: its muxer already writes through a temporary name of its own. One rule for both branches, and true whether or not this process is alive.
|
|
17
|
+
- **Fix**: The last piece of every run is provable. Under the successor rule nothing followed it, so it never was — which is the resume case that held a segment for 46 s and then answered 404 to a browser that had given up.
|
|
18
|
+
- **Fix**: Clearing up after a dead run is a name match against that run's own tag: no stretch to search, no bytes to judge, and no way to remove a complete piece another run closed. `services/encode/open-piece.js` did all three by guessing, and under the naming rule its guess would have removed a finished segment — the highest served name in a dead run's stretch is a piece that run closed. It is deleted, together with the session manager's own copy of it, which fetched the output's init bytes to judge a file by its contents. 135 lines of service code and 152 of tests for the guessing are gone; the manager is 83 lines shorter and 5 longer.
|
|
19
|
+
- **Fix**: One owner for "this piece is closed". A set of statements was kept beside the disk and told by whoever noticed a piece being produced — the same two-owner fault 2.80.11 removed from the coverage map, one layer down. A rename cannot go out of step with itself.
|
|
20
|
+
- **New**: `encode-plan on <output>` carries every term the decision was made from: `[speed=4.45x firstByte=1.26s kill=0.04s refetch=0.000s/s maxRuns=2 live=1]`. An interval says what was decided; only these say why. A decision of this plan is `delay + (index - at) / rate + madeBetween * refetch` against a deadline, so a recorded decision without the rate can be re-read and not recomputed — which is exactly what happened with the one-piece intervals of 2026-09-08, where the rate was substituted six times from the speeds the session reported elsewhere and none of them gave the answer the plan had given. A zero in the last three is a measurement nobody has taken, not a free operation, and it is printed so that reading it as free is a choice.
|
|
21
|
+
- **Chore**: 965 checks pass, biome clean. NOT yet seen in the field. What the next session must show: `making-*` files present in an output's directory while it is being written and none left after it; no `bufferAppendError` following a segment served twice at different sizes; and the terms above beside every placement.
|
|
22
|
+
|
|
1
23
|
## 2.80.18
|
|
2
24
|
|
|
3
25
|
- **Fix**: The `Infinity` is gone, and with it a double count I had introduced. The measured "time to a first piece" already CONTAINS one piece's encoding, and I was adding another; separated, the two scale differently — a piece costs more when encoders share the machine, a spawn does not. So a fresh encoder owes `spawn overhead + the piece at the rate in force`, a moved one owes the kill and then the same. With nothing measured the overhead is zero and a fresh encoder owes exactly one piece, which is the floor and is DERIVED rather than chosen: a piece cannot appear before it is encoded, and how fast this host encodes is measured before any viewer exists. The `Infinity` was an exception in a model that needs none. Verified by simulation over eighty ticks against the map's real shape — zero moves and zero one-piece intervals at a budget of one encoder and of three.
|
|
@@ -321,10 +321,25 @@ places three.
|
|
|
321
321
|
|
|
322
322
|
```
|
|
323
323
|
encode-plan on <output>: start #58..#481, stop #?..#?
|
|
324
|
+
[speed=4.45x firstByte=1.26s kill=0.04s refetch=0.000s/s maxRuns=2 live=1]
|
|
324
325
|
```
|
|
325
326
|
|
|
326
|
-
Every action with its INTERVAL, which is what a run is
|
|
327
|
-
does something — a session
|
|
327
|
+
Every action with its INTERVAL, which is what a run is, and then every term the
|
|
328
|
+
decision was made from. Printed on any pass that does something — a session
|
|
329
|
+
where nothing changes says nothing.
|
|
330
|
+
|
|
331
|
+
The terms are there because an interval says WHAT was decided and only these say
|
|
332
|
+
WHY. A decision of this plan is
|
|
333
|
+
|
|
334
|
+
delay + (index - at) / rate + madeBetween * refetch against a deadline
|
|
335
|
+
|
|
336
|
+
so a recorded decision without the rate can be re-read and not recomputed. That
|
|
337
|
+
is not hypothetical: the one-piece intervals of 2026-09-08 were diagnosed by
|
|
338
|
+
substituting the rate from the speeds the session reported elsewhere — six
|
|
339
|
+
different figures, none of which reproduced the answer the plan had given.
|
|
340
|
+
|
|
341
|
+
A zero in `firstByte`, `kill` or `refetch` is a measurement nobody has taken, not
|
|
342
|
+
a free operation. It is printed so that reading it as free is a choice.
|
|
328
343
|
|
|
329
344
|
It was missing, and its absence cost three wrong diagnoses of one field session.
|
|
330
345
|
The line printed the windows, the budget and where the live runs stood; the
|
|
@@ -336,6 +351,35 @@ reasons printed beside them read as moves back and forth, so the fault was read
|
|
|
336
351
|
as an oscillating placement three times over. An interval of one segment turns
|
|
337
352
|
the protection against two encoders writing one name into a mill for processes.
|
|
338
353
|
|
|
354
|
+
## What proves a segment is finished
|
|
355
|
+
|
|
356
|
+
Its NAME, and there is nothing else. A piece being written is called
|
|
357
|
+
`making-<from>-00042.mp4` — the tag is the first number of the stretch its run
|
|
358
|
+
was given — and it takes `segment-00042.mp4` when the encoder says it has closed
|
|
359
|
+
it, which it does on a channel of its own (`-segment_list pipe:3`). Making it
|
|
360
|
+
servable is therefore one rename inside one directory, performed by the store
|
|
361
|
+
because the store owns the disk. The `hls` branch needs nothing extra: its muxer
|
|
362
|
+
writes through a temporary name of its own, so its files appear under their final
|
|
363
|
+
name whole.
|
|
364
|
+
|
|
365
|
+
Three things follow, and each replaced a guess:
|
|
366
|
+
|
|
367
|
+
1. **a request can never reach a half-written piece.** Closure used to be
|
|
368
|
+
inferred from the NEXT number existing — sound for one writer walking forward,
|
|
369
|
+
false the moment two runs share an output, which is what the plan gives an
|
|
370
|
+
output whenever it places a second encoder. Field 2026-09-08:
|
|
371
|
+
`segment-00057.mp4` served at 2 268 361 bytes and then at 4 510 940, exactly
|
|
372
|
+
half; the browser appended the half and refused the whole for the rest of the
|
|
373
|
+
session, with the picture frozen at 319.66 s;
|
|
374
|
+
2. **the last piece of a run is provable.** Under the successor rule nothing
|
|
375
|
+
followed it, so it never was — the resume case that held one segment for 46 s
|
|
376
|
+
and then answered 404;
|
|
377
|
+
3. **clearing up after a dead run is a name match.** Its unfinished pieces are
|
|
378
|
+
the ones carrying its own tag: no stretch to search, no bytes to judge, and no
|
|
379
|
+
way to remove a complete piece somebody else closed. `services/encode/
|
|
380
|
+
open-piece.js` did all three of those by guessing and is gone, along with the
|
|
381
|
+
session manager's copy of it.
|
|
382
|
+
|
|
339
383
|
## What is checked
|
|
340
384
|
|
|
341
385
|
`test/one-authority.test.js` holds the shape: one caller of `#startEncodeRun`,
|
|
@@ -348,3 +392,8 @@ viewer registry, the real `LiveOutputs` and the real `PriorityOrchestrator`.
|
|
|
348
392
|
|
|
349
393
|
`test/encode-plan.test.js` holds the arithmetic, including that every encoder
|
|
350
394
|
stops when nobody is watching the output.
|
|
395
|
+
|
|
396
|
+
`test/segment-store.test.js` holds the naming rule: a piece under its served
|
|
397
|
+
name is finished — the last one of a run included — one under a working name is
|
|
398
|
+
not and cannot be reached, closing it is one rename, and clearing up after one
|
|
399
|
+
run leaves every other run's work alone.
|
package/package.json
CHANGED
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
# Каждый кодировщик запускается дважды, и второй процесс никто не убивает
|
|
2
|
+
|
|
3
|
+
Разбор восьми полевых сессий 8-10 сентября 2026: журнал прокси
|
|
4
|
+
(`/data/proxy.log` + `proxy.log.1`, 188 634 строки, версии 2.80.18 и 2.80.19) и
|
|
5
|
+
журнал браузера с дроплета (80 055 строк, восемь подписей сессий). Каждое
|
|
6
|
+
утверждение помечено тем, откуда взято: ЗАМЕРЕНО (число из журнала), ПРОЧИТАНО
|
|
7
|
+
(из исходников), ВЫВЕДЕНО (следствие двух первых).
|
|
8
|
+
|
|
9
|
+
## 1. Причина: две власти вызывают `start()` на одном прогоне
|
|
10
|
+
|
|
11
|
+
ПРОЧИТАНО. Прогон строится и ЗАПУСКАЕТСЯ в одном вызове:
|
|
12
|
+
`EncodeOrchestrator.#placeAt` → `makeRun` → `hls-session-manager.#makeRunAt`
|
|
13
|
+
(:3782) → `#startEncodeRun` (:5361), и последней строкой этой функции стоит
|
|
14
|
+
`run.start(because)` (:5546). Вернувшись, оркестратор вызывает
|
|
15
|
+
`run.start(because)` ещё раз (`EncodeOrchestrator.js:576`).
|
|
16
|
+
|
|
17
|
+
`EncodeRun.start()` (`EncodeRun.js:360`) защиты от повторного вызова не имеет:
|
|
18
|
+
строит аргументы заново, порождает второй процесс и ПЕРЕЗАПИСЫВАЕТ
|
|
19
|
+
`this.#process`. Обработчики первого процесса остаются подписанными на тот же
|
|
20
|
+
объект прогона.
|
|
21
|
+
|
|
22
|
+
Обе строки появились 2026-09-04, в двух соседних коммитах: `a09b278` («A run is
|
|
23
|
+
an object») поставила вызов в менеджере, `53785f2` («Encoders become classes») —
|
|
24
|
+
в оркестраторе.
|
|
25
|
+
|
|
26
|
+
ЗАМЕРЕНО, по переходам состояния за оба журнала:
|
|
27
|
+
|
|
28
|
+
```
|
|
29
|
+
207 IDLE --SPAWNED--> STARTING
|
|
30
|
+
207 STARTING --SPAWNED--> STARTING
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Ровно поровну. Двойной запуск происходит на КАЖДОМ прогоне без исключения: 207
|
|
34
|
+
прогонов, 414 процессов ffmpeg.
|
|
35
|
+
|
|
36
|
+
Ни один тест этого не ловит: в `test/encode-orchestrator.test.js` и соседних
|
|
37
|
+
`makeRun` — заглушка, которая прогон не запускает, поэтому вызов в менеджере не
|
|
38
|
+
исполняется ни разу.
|
|
39
|
+
|
|
40
|
+
## 2. Остановка убивает только второй процесс
|
|
41
|
+
|
|
42
|
+
ПРОЧИТАНО. `EncodeRun.stop()` шлёт SIGTERM в `this.#process` — то есть во
|
|
43
|
+
второй. Первый не адресуем ниоткуда: ссылки на него не осталось.
|
|
44
|
+
|
|
45
|
+
ЗАМЕРЕНО. Прогон `#453..#562` выхода `…grid=kf@38:video-only:v=38/copy`
|
|
46
|
+
остановлен в 09:12:01.178; его ffmpeg писал в поток ошибок до **09:13:46.938 —
|
|
47
|
+
105 секунд спустя**. Прогон `#463..#562` остановлен 09:12:10.204, последняя
|
|
48
|
+
строка 09:13:50.038 — 100 секунд спустя. Осиротевший процесс живёт до конца
|
|
49
|
+
своего отрезка, продолжая декодировать, читать из роя и писать файлы.
|
|
50
|
+
|
|
51
|
+
ЗАМЕРЕНО. Число РАЗНЫХ отрезков, писавших в поток ошибок в течение одной
|
|
52
|
+
секунды, на ОДНОМ выходе: 11 (09:13:31), 10 (09:13:32), 10 (09:13:34), 10
|
|
53
|
+
(09:13:05). Машина — CM4, четыре ядра. Бюджет в это время печатал `maxRuns=3`, и
|
|
54
|
+
машина состояний насчитывала не более трёх живых прогонов: **восемь из
|
|
55
|
+
одиннадцати процессов не видел никто.**
|
|
56
|
+
|
|
57
|
+
## 3. Что это ломает
|
|
58
|
+
|
|
59
|
+
### 3.1. Доказательство завершённости куска обойдено
|
|
60
|
+
|
|
61
|
+
Два процесса одного прогона получают ОДИН И ТОТ ЖЕ аргумент вывода —
|
|
62
|
+
`making-<метка>-%05d.mp4`, где метка равна первому номеру отрезка. Метка задумана
|
|
63
|
+
как имя прогона («отрезки не пересекаются, значит два живых прогона одного выхода
|
|
64
|
+
начинаются с разных номеров по построению»). Для двух процессов ОДНОГО прогона
|
|
65
|
+
это неверно: метка совпадает, и они пишут одно имя.
|
|
66
|
+
|
|
67
|
+
ЗАМЕРЕНО: **1105 строк** `segment store: could not publish making-…: ENOENT` —
|
|
68
|
+
один из двух переименовал файл, второму переименовывать нечего. По меткам: 904 на
|
|
69
|
+
`making-0`, остальные разбросаны по всем выходам и всем сессиям.
|
|
70
|
+
|
|
71
|
+
ВЫВЕДЕНО: это ровно та порча, ради которой 2.80.19 вводил рабочее имя — два
|
|
72
|
+
писателя на одно имя. Правка не отменена, она обойдена изнутри.
|
|
73
|
+
|
|
74
|
+
ЗАМЕРЕНО, что порча жива после 2.80.19: сессия `8cddc6f8` (10.09, 09:15:04) —
|
|
75
|
+
`fatal: bufferAppendError` одновременно на звуке (`segment-00072.mp4`) и на
|
|
76
|
+
картинке (`segment-00071.mp4`), `currentTime=712.61`. Сессия кончилась.
|
|
77
|
+
|
|
78
|
+
### 3.2. Механизм имени сам по себе исправен
|
|
79
|
+
|
|
80
|
+
ЗАМЕРЕНО на хосте дополнения: ffmpeg с `-f segment -segment_list pipe:3
|
|
81
|
+
-segment_list_flags +live` и шаблоном `making-0-%05d.mp4` пишет в четвёртый канал
|
|
82
|
+
имена `making-0-00000.mp4`, `…-00001`, `…-00002`, и файлы под этими именами лежат
|
|
83
|
+
на диске. ENOENT возникает не из-за имени и не из-за канала.
|
|
84
|
+
|
|
85
|
+
### 3.3. Двойное чтение из роя
|
|
86
|
+
|
|
87
|
+
ВЫВЕДЕНО. Каждый процесс открывает свой `/stream` (свой `createFragmentReader`),
|
|
88
|
+
поэтому читателей хранилища кусков вдвое больше, чем прогонов. Это спусковой
|
|
89
|
+
крючок тупика с закреплёнными кусками, где 2026-09-07 торрент был уничтожен при
|
|
90
|
+
четырёх читателях.
|
|
91
|
+
|
|
92
|
+
### 3.4. Процессорное время и задержки
|
|
93
|
+
|
|
94
|
+
ЗАМЕРЕНО. Ответы на опрос прогресса (тело около 500 байт): медиана 22 мс, p90
|
|
95
|
+
**2229 мс**, максимум **18 852 мс**; 362 ответа из 3516 дольше двух секунд.
|
|
96
|
+
ЗАМЕРЕНО. Предложение качества в те же минуты: `not offering 540p=0.10x
|
|
97
|
+
480p=0.11x 360p=0.12x 240p=0.13x`.
|
|
98
|
+
|
|
99
|
+
## 4. Что найдено рядом и причиной не является
|
|
100
|
+
|
|
101
|
+
### 4.1. Зритель простоял 44 минуты, и ни один прибор этого не сосчитал
|
|
102
|
+
|
|
103
|
+
Сессия `39c188b5` (08.09, 20:38-21:28). Зритель перемотал на 551.1 с, затем на
|
|
104
|
+
579.4 с. Картинка встала на 596.8 с, `readyState=1`, и до конца сессии не
|
|
105
|
+
сдвинулась: 528 одинаковых строк `decode t=596.8s frames=11787`.
|
|
106
|
+
|
|
107
|
+
ЗАМЕРЕНО: буфер держал `video=[…413..549.091] [859.192..886.136]` — перемотка
|
|
108
|
+
пришлась в дыру между 549.1 и 859.2. Браузер запросил за это время 283 куска, но
|
|
109
|
+
класть их в 596.8 было нечего.
|
|
110
|
+
|
|
111
|
+
ЗАМЕРЕНО: счётчик «сколько стояла картинка» не напечатал НИ ОДНОЙ строки за эту
|
|
112
|
+
сессию, а переход в состояние ожидания случился один раз. Счётчик закрывается
|
|
113
|
+
только по возобновлению, поэтому перерыв, который не кончился, не записывается
|
|
114
|
+
вовсе — худший случай невидим по построению.
|
|
115
|
+
|
|
116
|
+
ЗАМЕРЕНО: 42 `levelLoadError` за 48 минут при том, что прокси в те же секунды
|
|
117
|
+
отдавал `index.m3u8` (17 539 байт, `fetchMs=8-10`). Обе стороны о запросе
|
|
118
|
+
плейлиста говорят разное.
|
|
119
|
+
|
|
120
|
+
ЗАМЕРЕНО: браузер запрашивал `/v/540/index.m3u8` и `/v/480/index.m3u8`, декодируя
|
|
121
|
+
1920x1080 — hls.js по-прежнему сам ходит по ступеням при ошибках, вопреки
|
|
122
|
+
закреплению.
|
|
123
|
+
|
|
124
|
+
### 4.2. Куча главного потока выросла до 845 МБ за 26 секунд
|
|
125
|
+
|
|
126
|
+
ЗАМЕРЕНО (10.09): 09:15:20 heap=101 МБ, 09:15:31 — 378 МБ, 09:15:35 — 589 МБ,
|
|
127
|
+
09:15:46 — **813 МБ**, `rss=1940MB`. Написаны два снимка кучи процесса —
|
|
128
|
+
`heap-process-…-451543040` (431 МБ, 09:09) и `heap-process-…-885628928` (845 МБ,
|
|
129
|
+
09:15:44), оба в `/data`. **Их никто не открывал.** Это то самое чтение, которого
|
|
130
|
+
ждёт работа о необъяснённом росте памяти.
|
|
131
|
+
|
|
132
|
+
ЗАМЕРЕНО: после конца сессии куча падает до 35 МБ, а `rss` остаётся 979-981 МБ
|
|
133
|
+
четыре часа подряд без единого зрителя.
|
|
134
|
+
|
|
135
|
+
### 4.3. Ошибки разбора потока
|
|
136
|
+
|
|
137
|
+
ЗАМЕРЕНО: 20 294 строки `missing picture in access unit` и 20 118 строк `Invalid
|
|
138
|
+
NAL unit size` — 89 % журнала. Все на ветке КОПИРОВАНИЯ картинки файлов `.mp4`
|
|
139
|
+
(`grid=kf@38` — 629 строк с адресом, `kf@33` — 185, `kf@46` — 94). Причина не
|
|
140
|
+
установлена. Проверено, что общего состояния чтения у двух процессов нет: каждый
|
|
141
|
+
запрос `/stream` создаёт свой читатель.
|
|
142
|
+
|
|
143
|
+
### 4.4. Прогоны, не сделавшие ничего
|
|
144
|
+
|
|
145
|
+
ЗАМЕРЕНО: 153 прогона остановлены планом, из них **68 не произвели ни одного
|
|
146
|
+
куска**. Медиана жизни остановленного прогона 9636 мс, p90 26 719 мс, суммарно
|
|
147
|
+
1885 секунд. Причины остановки: 83 — «standing at #N scores worse than standing
|
|
148
|
+
at #M», 49 — «the film is no worse off without it». Довели свой отрезок до конца
|
|
149
|
+
48 из 207.
|
|
150
|
+
|
|
151
|
+
### 4.5. Мелочи, каждая с местом
|
|
152
|
+
|
|
153
|
+
1. Поле `live=` в строке плана печатается пустым 89 раз из 211 (`maxRuns=3 live=`).
|
|
154
|
+
2. `/tmp/torrent-tv-hls` — 5,0 ГБ готовых кусков от сессий, кончившихся часами
|
|
155
|
+
раньше; 3272 файла в десяти каталогах. Ничего не подметает.
|
|
156
|
+
3. `/data` — 3,3 ГБ, из них слепок памяти на 4,4 ГБ от 07.09 и 365 МБ снимков
|
|
157
|
+
кучи.
|
|
158
|
+
4. Предложение качества в первую секунду сессии переворачивается: 19:39:40.568
|
|
159
|
+
`not offering 1080p=0.86x (offering 720p 540p 480p 360p 240p)`, 19:39:41.058
|
|
160
|
+
`not offering 720p=0.24x … (offering 1080p)`. Первое читает копируемую картинку
|
|
161
|
+
как перекодирование.
|
|
162
|
+
|
|
163
|
+
## 5. Чего этот разбор НЕ устанавливает
|
|
164
|
+
|
|
165
|
+
1. почему ffmpeg на ветке копирования выдаёт ошибки разбора — связь с двойным
|
|
166
|
+
запуском не показана;
|
|
167
|
+
2. что именно держит 945 МБ резидентной памяти при куче в 35 МБ — снимки есть,
|
|
168
|
+
они не прочитаны;
|
|
169
|
+
3. почему браузер сообщает `levelLoadError`, когда прокси отдал плейлист без
|
|
170
|
+
ошибки;
|
|
171
|
+
4. откуда в буфере взялся оторванный остров 859..886 с.
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file One owner of the disk, read by everything that takes any of it.
|
|
3
|
+
*
|
|
4
|
+
* Three things on this proxy write to the same disk and, until now, each
|
|
5
|
+
* decided for itself how much it could take:
|
|
6
|
+
*
|
|
7
|
+
* - the segments an encoder produces, bounded by a quarter of what was free
|
|
8
|
+
* plus a floor of 2 GB, both numbers chosen out of nothing;
|
|
9
|
+
* - the pieces the memory store spills, bounded by nothing at all — 14 400 MB
|
|
10
|
+
* written in one fifty-minute viewing, field 2026-08-31;
|
|
11
|
+
* - the diagnostics we keep on purpose (core dumps, heap snapshots, packet
|
|
12
|
+
* captures), each bounded by a COUNT and none by a size: on the addon host
|
|
13
|
+
* two dumps and five snapshots came to 3.2 GB.
|
|
14
|
+
*
|
|
15
|
+
* Each read the free space as though it were the only claimant, so three
|
|
16
|
+
* ceilings each stood for the whole disk. This is the one place that reads it,
|
|
17
|
+
* and what it hands out is a share.
|
|
18
|
+
*
|
|
19
|
+
* THE RULE IS THE ONE MEMORY ALREADY USES, on the other reading: what is free
|
|
20
|
+
* now, plus what we already hold, less what everything that is not us has been
|
|
21
|
+
* seen to need. Nothing is a fraction chosen by hand.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import { OtherDemand, divideAllowance } from "../piece-store/allowance.js";
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* A consumer of the disk.
|
|
28
|
+
*
|
|
29
|
+
* @typedef {object} DiskConsumer
|
|
30
|
+
* @property {string} name - What it is called in the reading.
|
|
31
|
+
* @property {() => number} held - What it holds right now, in bytes.
|
|
32
|
+
* @property {() => number} wanted - What it would take if it could. A consumer
|
|
33
|
+
* that cannot say asks for what it holds, which is the honest statement of a
|
|
34
|
+
* thing that only grows when something arrives.
|
|
35
|
+
* @property {(allowanceBytes: number) => void} allow - Told its share.
|
|
36
|
+
*/
|
|
37
|
+
|
|
38
|
+
export class DiskSpace {
|
|
39
|
+
/** @type {Map<string, DiskConsumer>} */
|
|
40
|
+
#consumers = new Map();
|
|
41
|
+
|
|
42
|
+
#otherDemand = new OtherDemand();
|
|
43
|
+
|
|
44
|
+
#readFree;
|
|
45
|
+
|
|
46
|
+
#logger;
|
|
47
|
+
|
|
48
|
+
/** The last division, for the reading. @type {{ name: string, held: number, allowed: number }[]} */
|
|
49
|
+
#last = [];
|
|
50
|
+
|
|
51
|
+
#freeBytes = 0;
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* @param {object} params
|
|
55
|
+
* @param {() => Promise<number | null>} params.readFree - What the machine
|
|
56
|
+
* says is free on the disk these consumers share.
|
|
57
|
+
* @param {{ info: (line: string) => void, warn?: (line: string) => void }} [params.logger]
|
|
58
|
+
*/
|
|
59
|
+
constructor({ readFree, logger = null }) {
|
|
60
|
+
this.#readFree = readFree;
|
|
61
|
+
this.#logger = logger;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Register a consumer. Registering twice under one name replaces it.
|
|
66
|
+
*
|
|
67
|
+
* @param {DiskConsumer} consumer
|
|
68
|
+
* @returns {void}
|
|
69
|
+
*/
|
|
70
|
+
register(consumer) {
|
|
71
|
+
this.#consumers.set(consumer.name, consumer);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* @param {string} name
|
|
76
|
+
* @returns {void}
|
|
77
|
+
*/
|
|
78
|
+
forget(name) {
|
|
79
|
+
this.#consumers.delete(name);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Read the disk, divide it, and tell each consumer its share.
|
|
84
|
+
*
|
|
85
|
+
* Called on the same timer that revises memory: a disk that fills while a
|
|
86
|
+
* film is playing must lower the ceilings, not keep ones taken when it was
|
|
87
|
+
* empty. That is the mistake memory made until 2026-08-28, and every disk
|
|
88
|
+
* ceiling on this proxy made until now.
|
|
89
|
+
*
|
|
90
|
+
* @returns {Promise<{ freeBytes: number, allowanceBytes: number, shares: { name: string, held: number, allowed: number }[] }>}
|
|
91
|
+
*/
|
|
92
|
+
async revise() {
|
|
93
|
+
const consumers = [...this.#consumers.values()];
|
|
94
|
+
if (consumers.length === 0) {
|
|
95
|
+
return { freeBytes: 0, allowanceBytes: 0, shares: [] };
|
|
96
|
+
}
|
|
97
|
+
const held = consumers.reduce((sum, consumer) => sum + Math.max(0, consumer.held()), 0);
|
|
98
|
+
const free = await this.#readFree();
|
|
99
|
+
// WITHOUT A READING, NOTHING IS ALLOWED TO GROW. A disk whose free space
|
|
100
|
+
// cannot be read is not a disk with room; answering "unbounded" there is how
|
|
101
|
+
// the spill file came to have no limit in the first place.
|
|
102
|
+
this.#freeBytes = Number.isFinite(free) && free !== null ? Math.max(0, free) : 0;
|
|
103
|
+
const reserve = this.#otherDemand.note(this.#freeBytes, held);
|
|
104
|
+
const allowance = Math.max(0, this.#freeBytes + held - reserve);
|
|
105
|
+
const shares = divideAllowance(
|
|
106
|
+
consumers.map((consumer) => Math.max(0, consumer.wanted())),
|
|
107
|
+
allowance
|
|
108
|
+
);
|
|
109
|
+
this.#last = consumers.map((consumer, position) => ({
|
|
110
|
+
name: consumer.name,
|
|
111
|
+
held: consumer.held(),
|
|
112
|
+
allowed: shares[position]
|
|
113
|
+
}));
|
|
114
|
+
for (const [position, consumer] of consumers.entries()) {
|
|
115
|
+
consumer.allow(shares[position]);
|
|
116
|
+
}
|
|
117
|
+
return { freeBytes: this.#freeBytes, allowanceBytes: allowance, shares: this.#last };
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* One line: what the disk has, what each consumer holds, and what it may.
|
|
122
|
+
*
|
|
123
|
+
* Said because "why is there no room" is otherwise a question no log can
|
|
124
|
+
* answer — the three ceilings were computed in three places and none of them
|
|
125
|
+
* was printed beside the others.
|
|
126
|
+
*
|
|
127
|
+
* @returns {string}
|
|
128
|
+
*/
|
|
129
|
+
describe() {
|
|
130
|
+
if (this.#last.length === 0) {
|
|
131
|
+
return "disk: nothing has claimed any yet";
|
|
132
|
+
}
|
|
133
|
+
const parts = this.#last.map(
|
|
134
|
+
(share) => `${share.name} ${megabytes(share.held)} of ${megabytes(share.allowed)}`
|
|
135
|
+
);
|
|
136
|
+
return `disk: ${megabytes(this.#freeBytes)} free; ${parts.join(", ")}`;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* @param {number} bytes
|
|
142
|
+
* @returns {string}
|
|
143
|
+
*/
|
|
144
|
+
function megabytes(bytes) {
|
|
145
|
+
return `${Math.round(Math.max(0, bytes) / (1024 * 1024))}MB`;
|
|
146
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file Who takes disk, and how each of them is told its share.
|
|
3
|
+
*
|
|
4
|
+
* Kept apart from the owner because the owner knows nothing about this proxy —
|
|
5
|
+
* it reads a number, divides it and hands out shares — and kept out of the
|
|
6
|
+
* session manager because a list of claimants is not a fact about a session.
|
|
7
|
+
*
|
|
8
|
+
* Two claimants today. The diagnostics are a third and are not here yet: they
|
|
9
|
+
* are bounded by a COUNT and never by a size, which on the addon host let two
|
|
10
|
+
* core dumps and five heap snapshots come to 3.2 GB, and they cannot simply be
|
|
11
|
+
* thrown away when space is short — a dump is the only evidence of the death it
|
|
12
|
+
* records. That needs a rule of its own.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { DiskSpace } from "./DiskSpace.js";
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Build the owner of the disk and register everything that takes any of it.
|
|
19
|
+
*
|
|
20
|
+
* @param {object} params
|
|
21
|
+
* @param {{ root: string, stats: () => { bytes: number } }} params.segmentStore
|
|
22
|
+
* @param {{ spilledBytes?: number, allowSpillBytes?: (bytes: number) => unknown }} [params.torrentPool]
|
|
23
|
+
* @param {(directory: string) => Promise<number | null>} params.readFree
|
|
24
|
+
* @param {{ info: Function, warn?: Function }} [params.logger]
|
|
25
|
+
* @returns {{ revise: () => Promise<unknown>, segmentBytes: () => number, describe: () => string }}
|
|
26
|
+
* What the segments may hold is asked for rather than pushed: zero until the
|
|
27
|
+
* first revision, and zero stops growth rather than licensing it.
|
|
28
|
+
*/
|
|
29
|
+
export function wireDiskSpace({ segmentStore, torrentPool, readFree, logger }) {
|
|
30
|
+
const space = new DiskSpace({ readFree: () => readFree(segmentStore.root), logger });
|
|
31
|
+
let segmentBytes = 0;
|
|
32
|
+
space.register({
|
|
33
|
+
name: "segments",
|
|
34
|
+
held: () => segmentStore.stats().bytes,
|
|
35
|
+
// The whole of every film anybody is watching. There is no smaller honest
|
|
36
|
+
// answer, so it asks for everything and is cut in proportion like the rest.
|
|
37
|
+
wanted: () => Number.MAX_SAFE_INTEGER,
|
|
38
|
+
allow: (bytes) => {
|
|
39
|
+
segmentBytes = bytes;
|
|
40
|
+
}
|
|
41
|
+
});
|
|
42
|
+
if (typeof torrentPool?.allowSpillBytes === "function") {
|
|
43
|
+
// The pieces the memory store spills. They live on the torrent thread, so
|
|
44
|
+
// the share travels the channel that already carries everything else, and
|
|
45
|
+
// the reply says what they hold — one exchange, both directions.
|
|
46
|
+
space.register({
|
|
47
|
+
name: "spilled pieces",
|
|
48
|
+
held: () => torrentPool.spilledBytes ?? 0,
|
|
49
|
+
wanted: () => Number.MAX_SAFE_INTEGER,
|
|
50
|
+
allow: (bytes) => {
|
|
51
|
+
void torrentPool.allowSpillBytes?.(bytes);
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
return {
|
|
56
|
+
revise: () => space.revise(),
|
|
57
|
+
segmentBytes: () => segmentBytes,
|
|
58
|
+
describe: () => space.describe()
|
|
59
|
+
};
|
|
60
|
+
}
|
|
@@ -181,9 +181,11 @@ export class EncodeRun {
|
|
|
181
181
|
* @param {{ info: (line: string) => void, warn: (line: string) => void, error?: (line: string) => void }} params.logger
|
|
182
182
|
* @param {() => number} [params.now]
|
|
183
183
|
* @param {(ended: RunEnded) => void} [params.onEnded]
|
|
184
|
-
* @param {(name: string) =>
|
|
185
|
-
* of every piece the encoder has
|
|
186
|
-
* names it on its own channel
|
|
184
|
+
* @param {(name: string) => string | null} [params.onClosed] - Called with the
|
|
185
|
+
* WORKING name of every piece the encoder has finished writing, as the
|
|
186
|
+
* encoder itself names it on its own channel, and answers with the name that
|
|
187
|
+
* piece is served under — because making it servable is a rename, and only
|
|
188
|
+
* whoever owns the disk can perform one.
|
|
187
189
|
* @param {(progress: { processedSeconds: number | null, speed: string | null }) => void} [params.onProgress]
|
|
188
190
|
* Called for every `-progress` report. Seconds count from the START OF THIS
|
|
189
191
|
* RUN on both branches — neither `-output_ts_offset` nor `-copyts` changes
|
|
@@ -199,6 +201,9 @@ export class EncodeRun {
|
|
|
199
201
|
* kept so a failure can quote what produced it.
|
|
200
202
|
* @param {boolean} [params.usesExplicitCuts] - Whether this run cuts at times
|
|
201
203
|
* it was given, which decides how a segment is judged finished.
|
|
204
|
+
* @param {string} params.because - Why this encoder is being put on the
|
|
205
|
+
* machine, in words. Recorded with the argument list: a start whose cause is
|
|
206
|
+
* not written down cannot be told from any other when several runs exist.
|
|
202
207
|
* @param {(name: string) => number | null} [params.indexOfName] - The number
|
|
203
208
|
* a closed piece's name carries. How a piece is named belongs to the format
|
|
204
209
|
* that writes it, so it arrives as a plain function rather than this class
|
|
@@ -220,7 +225,8 @@ export class EncodeRun {
|
|
|
220
225
|
inputUnavailable,
|
|
221
226
|
argsDescribed = "",
|
|
222
227
|
usesExplicitCuts = false,
|
|
223
|
-
indexOfName
|
|
228
|
+
indexOfName,
|
|
229
|
+
because = "no reason was given"
|
|
224
230
|
}) {
|
|
225
231
|
this.address = address;
|
|
226
232
|
this.encoder = encoder;
|
|
@@ -242,6 +248,9 @@ export class EncodeRun {
|
|
|
242
248
|
this.indexOfName = typeof indexOfName === "function" ? indexOfName : () => null;
|
|
243
249
|
/** The last thing ffmpeg said on stderr, which is what a failure is explained by. */
|
|
244
250
|
this.lastError = "";
|
|
251
|
+
// EXISTING IS RUNNING. There is no moment at which a built run is not yet a
|
|
252
|
+
// process, so there is no second act for two owners to perform.
|
|
253
|
+
this.#begin(because);
|
|
245
254
|
}
|
|
246
255
|
|
|
247
256
|
/** @returns {string} */
|
|
@@ -346,7 +355,22 @@ export class EncodeRun {
|
|
|
346
355
|
}
|
|
347
356
|
|
|
348
357
|
/**
|
|
349
|
-
*
|
|
358
|
+
* Put the process on the machine, and say why.
|
|
359
|
+
*
|
|
360
|
+
* PRIVATE, AND CALLED ONCE, from the constructor. It was public until
|
|
361
|
+
* 2026-09-10, and two places called it: the session manager built a run and
|
|
362
|
+
* started it, then handed it back to the orchestrator, which started it
|
|
363
|
+
* again. Every run of every session therefore had TWO ffmpeg processes on one
|
|
364
|
+
* output writing one set of names — 207 runs against 414 spawns in the field
|
|
365
|
+
* logs of 08-10 September, without a single exception. Only the second was
|
|
366
|
+
* reachable afterwards, because this line overwrote the reference to the
|
|
367
|
+
* first, so `stop` killed one and the other ran on: measured 105 seconds past
|
|
368
|
+
* its own run's death, eleven processes writing at once on a four-core host
|
|
369
|
+
* whose budget said three.
|
|
370
|
+
*
|
|
371
|
+
* The guard against that is not a check but the absence of a second act: a
|
|
372
|
+
* run exists means its process is running, so there is nothing anybody can
|
|
373
|
+
* call twice.
|
|
350
374
|
*
|
|
351
375
|
* The reason is not decoration: a start whose cause is not recorded cannot be
|
|
352
376
|
* told from any other start when several runs exist at once, and the argument
|
|
@@ -355,7 +379,7 @@ export class EncodeRun {
|
|
|
355
379
|
*
|
|
356
380
|
* @param {string} because
|
|
357
381
|
*/
|
|
358
|
-
|
|
382
|
+
#begin(because) {
|
|
359
383
|
const args = this.buildArgs();
|
|
360
384
|
this.#startedAt = this.now();
|
|
361
385
|
this.logger.info(
|
|
@@ -473,8 +497,13 @@ export class EncodeRun {
|
|
|
473
497
|
if (name.length === 0) {
|
|
474
498
|
continue;
|
|
475
499
|
}
|
|
500
|
+
// ITS SERVED NAME, which is what whoever owns the disk gives it in answer.
|
|
501
|
+
// ffmpeg writes a piece under a working name and reports that; the piece
|
|
502
|
+
// becomes servable by being renamed, and everything below works in the
|
|
503
|
+
// name a request can actually ask for.
|
|
504
|
+
const served = this.onClosed(name) ?? name;
|
|
476
505
|
if (!this.#stopping) {
|
|
477
|
-
this.#provenName =
|
|
506
|
+
this.#provenName = served;
|
|
478
507
|
}
|
|
479
508
|
// WHAT THIS RUN HAS MADE IS THIS RUN'S OWN FACT, and this channel is where
|
|
480
509
|
// it learns it. It used to be told from outside, by whoever listed the
|
|
@@ -484,11 +513,10 @@ export class EncodeRun {
|
|
|
484
513
|
// (483 segment(s))", having produced none of them, and its head therefore
|
|
485
514
|
// described somebody else's work. Both the claim it holds and the cleanup
|
|
486
515
|
// after it read that head.
|
|
487
|
-
const index = this.indexOfName(
|
|
516
|
+
const index = this.indexOfName(served);
|
|
488
517
|
if (Number.isInteger(index)) {
|
|
489
518
|
this.noteProduced(index);
|
|
490
519
|
}
|
|
491
|
-
this.onClosed(name);
|
|
492
520
|
}
|
|
493
521
|
}
|
|
494
522
|
|