funoteka 0.1.0

package/dist/tags/read.js

@@ -0,0 +1,269 @@
+import { readFlac } from "./flac.js";
+import { readId3v1 } from "./id3v1.js";
+import { readId3v2 } from "./id3v2.js";
+import { mpegLength } from "./mpeg.js";
+import { readMp4 } from "./mp4.js";
+import { readOgg } from "./ogg.js";
+import { noTags } from "./types.js";
+/**
+ * Which reading of a file this code takes.
+ *
+ * The reader set below is not a constant — readers get fixed and new formats get
+ * added, and on 2026-09-12 a FLAC reader was found to have been misreading real
+ * files since it was written. A stored verdict describes a file *as read by a
+ * particular reading*, so when that reading changes, every verdict below it is
+ * no longer a verdict about the file.
+ *
+ * This number is what says so. It is written on every file the stage reads and
+ * is a reason to read a file again, exactly as `PROBE_METHOD` is for a probe row
+ * (`db/migrations/015_probe_method.sql`, and the same idea in `apply.ts`).
+ * Raising it re-reads the library once; without it a reader could be corrected
+ * and never reach a file that had not moved, which is precisely what happened
+ * when the Ogg reader landed and eighty `.ogg` files stayed empty
+ * (`db/migrations/016_tags_method.sql`).
+ *
+ * 2 — a picture carried as a comment stopped being written down as a tag. It
+ * arrives looking like any other name and value, so the eighty `.ogg` files put
+ * nineteen and a half megabytes of base64 into `file_tag`, one of them in a
+ * single value of 562 KB. Correcting that is a change of reading like any other,
+ * and a file it does not reach keeps its base64 for good.
+ *
+ * 3 — ID3v1 is read. Its block sits at the *other* end of an mp3 from the v2
+ * block and had been read by nobody, so a file whose names are only there was
+ * read as a bare stream with no tags and no complaint. 273 files of this
+ * collection are exactly that, and 489 more carry both blocks — the older one
+ * filling names the newer left unstated.
+ *
+ * 4 — and the verdict stored for such a file is now about the text that
+ * survives. 3 weighed the whole older block, so an mp3 whose v2 block is clean
+ * UTF-8 and whose v1 block is Latin-1 was filed as a guess on the strength of
+ * titles nothing keeps: 328 findings on this collection about a name the newer
+ * block had already answered for. The reading did not change; what is recorded
+ * about it did, which is the same kind of change and reaches a stored row the
+ * same way.
+ *
+ * 5 — an ID3 tag yields every frame the standard calls text rather than the nine
+ * the reader used to hold, a comment is read, and the date is composed from the
+ * frames §4.2.1 splits it across (task:2737). Measured by running the reader
+ * over the collection before and after: 447 of its 703 tagged mp3 gain at least
+ * one name, and no name or value a file already had is lost. Without this number
+ * none of that reaches a file that does not move — which was measured too: a
+ * scan after the change but before this bump read *no* files at all.
+ *
+ * 6 — an MP4's `rtng` atom is read, which is the content rating the
+ * specification calls `explicitStatus` (task:2866). 1425 of this collection's
+ * files are `.m4a`, and the atom was previously dropped without a word: through
+ * the pair branch (`trkn`/`disk`) a one-byte atom fails the length check, and
+ * the pass-through branch does not exist for MP4 at all. Measured after the
+ * reader landed: fifteen of fifty-two sampled `.m4a` files carry the atom, and
+ * the field was empty for every one of them.
+ *
+ * **This entry exists because the bump was missed.** The reader was corrected in
+ * `ca3af89` and this number was left at 5, so a rescan re-read nothing — 3350
+ * files sat at `tags_method = 5` and stayed there, the one `rtng` row in the
+ * whole meta layer belonging to a file that had moved for other reasons. The
+ * defect was found by the cluster's acceptance review and not by a test, which
+ * is what synthetic-only coverage of this field buys.
+ *
+ * 7 — the MPEG frame walk reports the sample rate and the channel count, which
+ * it had always read and never carried out (task:2910). Nothing about the
+ * reading changed; what is *recorded* about it did, which is exactly the case
+ * entry 4 above is about and reaches a stored row the same way.
+ *
+ * Measured on the live base before the change: of the collection's mp3, **728**
+ * had a codec and no channel count, because the reader answered the length and
+ * nothing else — and `alreadyIs` cannot check a client's `maxAudioChannels`
+ * against a number that is not there, so every one of those files was
+ * transcoded whole rather than served: **2423.7 ms** cold against 10.3 ms for
+ * the byte copy, on a request that needed no work at all. This bump is what
+ * takes them back, and without it a rescan would re-read none of them.
+ *
+ * 8 — the MP4 reader names the codec its sound track's sample description
+ * states (task:2910, the other half). Same shape of change as 5 and 7 and the
+ * same reason: what is *recorded* about the file changed, and the 1425 `.m4a`
+ * of this collection are the files it is about. Its own probe-method bump was
+ * needed beside this one and for the reason recorded there.
+ *
+ * 9 — that same reader states the channel count and the sample rate beside the
+ * codec, out of the same 28-byte header it was already standing on. Three
+ * entries and three bumps in one afternoon is what working the same hole from
+ * one end looks like; each is a real change to what a stored row says, and the
+ * number is the only thing that can say so to a file that has not moved.
+ *
+ * 10 — the stage writes a second thing beside `file_tag`: `file_tag_first`, the
+ * eight tags a listing shows, one row a file (task:2925). **This is the entry
+ * that makes that row self-healing, and it was nearly not written.**
+ *
+ * The row is derived from `file_tag` and written by the stage, and the migration
+ * that first filled it runs once. So a build that predates the row — a `git
+ * revert`, which this runbook calls safe, or a scan still running the old code
+ * while the daemon has been restarted onto the new — writes `file_tag` and no
+ * row. Nothing brings that file back: this build sees `tags_method` already at
+ * its own number and does not re-read it, and the migration will not run twice.
+ * The file answers empty in all eight fields for good, which is the same defect
+ * `task:2921` was: a client that normalises by ReplayGain, and a genre nobody is
+ * shown.
+ *
+ * Measured rather than reasoned — a fixture read by the build at `0fa535a` and
+ * then by this one: `tags 0 (0 files read)`, and the same bytes answering
+ * `{"artist":"Slipknot","replayGain":{}}` under one build and
+ * `{"genre":"Trance","artist":"VA","replayGain":{"trackGain":-6.62}}` under the
+ * other.
+ *
+ * Raising this number is what closes it, and it closes it *for every future
+ * mixed build* rather than for this one: a writer that does not know about the
+ * row writes an older number, so the next scan of this build re-reads the file
+ * and derives the row. The number is what the stage *writes*, not only what the
+ * reader reads — entries 4, 7, 8 and 9 say the same thing, and this is the first
+ * time the thing written was not a column of a row the stage already wrote.
+ */
+export const TAGS_METHOD = 10;
+/** Does the file say this, at this offset? */
+function magicAt(bytes, at, magic) {
+    if (at + magic.length > bytes.length)
+        return false;
+    for (let i = 0; i < magic.length; i += 1) {
+        if (bytes[at + i] !== magic.charCodeAt(i))
+            return false;
+    }
+    return true;
+}
+/**
+ * Read whatever a file's own bytes say about it.
+ *
+ * Dispatched on the bytes rather than the extension: a `.flac` holding
+ * something else is a normal thing for a collection to contain, and believing
+ * the name over the file is how a reader ends up parsing nonsense.
+ *
+ * Never throws, by design. A format nothing here understands, a file with no
+ * metadata blocks, and a half-copied download all answer the same way — with
+ * nothing — because the caller is a scan over fifty thousand files and one
+ * damaged one must not be able to stop it.
+ */
+export function readTags(bytes) {
+    if (magicAt(bytes, 0, 'fLaC'))
+        return readFlac(bytes);
+    // Ogg is dispatched on the same four bytes a reader synchronises on, and both
+    // of its codecs — Vorbis and Opus — keep their tags in the same comment
+    // packet, so one reader covers the two.
+    if (magicAt(bytes, 0, 'OggS'))
+        return readOgg(bytes);
+    // At four, not zero: an MP4 opens with the length of the box that names it,
+    // and `ftyp` is that box's type. Checking offset zero finds `ftyp` on nothing
+    // and misses every m4a there is.
+    if (magicAt(bytes, 4, 'ftyp'))
+        return readMp4(bytes);
+    if (magicAt(bytes, 0, 'ID3')) {
+        const tag = readId3v2(bytes);
+        // The tag states the names and the audio behind it states the length, and
+        // both come out of the one read. An ID3 block is followed by MPEG frames in
+        // every file this project meets, but nothing is assumed: if no frame can be
+        // found the duration is simply unknown, exactly as before.
+        const length = mpegLength(bytes, id3End(bytes));
+        const withOlder = olderTag(tag, bytes);
+        if (length === null)
+            return withOlder;
+        // Refused and unknown are different answers and the difference is carried
+        // up: this one has something behind it to measure, and a reader that failed
+        // to is worth telling someone who might not. What the frames turned out to
+        // be is carried up with it, because the tag block above never knew — and
+        // that is the whole of the audio format, not just its name: the rate and
+        // the channel count are in the same header the walk read (task:2910).
+        const format = {
+            codec: length.codec,
+            sampleRate: length.sampleRate,
+            channels: length.channels,
+        };
+        return length.refused
+            ? { ...withOlder, ...format, durationRefused: true }
+            : { ...withOlder, ...format, durationMs: length.durationMs };
+    }
+    // The ID3v1 block, which is the *other* end of the file and has nothing to do
+    // with whether a v2 block is at this one — a file may carry both, one, or
+    // neither.
+    const older = readId3v1(bytes);
+    // A bare MPEG stream — no v2 tag block at all, which for a collection rip is
+    // entirely ordinary and is not "a format nothing understands". An ID3v1 block
+    // is what a rip of that age usually has instead.
+    const bare = mpegLength(bytes, 0);
+    if (bare === null) {
+        // No frames found. With a v1 block the file is still understood — the
+        // container is the tag block, and its names are read even though nothing
+        // here can measure the audio. Without one there is nothing to say.
+        return older === null
+            ? noTags()
+            : { ...noTags('id3v1'), tags: older.tags, refusals: recover(older.refusals), encoding: older.encoding };
+    }
+    // The same header the length came from, so all three fields arrive together
+    // or not at all — see the note on the tag-block path above.
+    const format = { sampleRate: bare.sampleRate, channels: bare.channels };
+    const measured = bare.refused
+        ? { ...noTags('mpeg', bare.codec), ...format, durationRefused: true }
+        : { ...noTags('mpeg', bare.codec), ...format, durationMs: bare.durationMs };
+    return older === null
+        ? measured
+        : {
+            ...measured,
+            // No v2 block here, so the v1 names are the only statement of them —
+            // this is the container's own tag, not a fallback behind another.
+            container: 'id3v1',
+            tags: older.tags,
+            refusals: recover(older.refusals),
+            encoding: older.encoding,
+        };
+}
+/** A field list or nothing at all, so an empty array never reaches a caller. */
+function recover(refusals) {
+    return refusals.length === 0 ? undefined : refusals;
+}
+/**
+ * The v2 reading, with the v1 block beside the same file attached as a fallback.
+ *
+ * `fallbackTags` rather than appended to `tags`, and the stage is what acts on
+ * the difference — see `TagRead.fallbackTags` on why two blocks merged is a
+ * collaboration this project would then invent. What the older block *refused*
+ * is reported either way, because a field read and discarded is worth saying
+ * whatever became of the tag around it.
+ */
+function olderTag(tag, bytes) {
+    const older = readId3v1(bytes);
+    if (older === null)
+        return tag;
+    // Only the names the newer block does not speak are weighted, because only
+    // those are kept — the same rule the stage applies, and it has to be the same
+    // one: the reader says how sure it is of the text that survives, and text
+    // nobody uses is not the file's certainty. Weighting the whole older block
+    // instead put 328 encoding findings on this collection, every one of them
+    // about a title the v2 block had already answered for.
+    const newerNames = new Set(tag.tags.map((one) => one.name));
+    let encoding = tag.encoding;
+    for (const item of older.tags) {
+        if (newerNames.has(item.name))
+            continue;
+        const verdict = older.verdicts.get(item.name);
+        if (verdict !== undefined && (encoding === null || verdict.confidence < encoding.confidence)) {
+            encoding = verdict;
+        }
+    }
+    // Two blocks can each have refused something, and both are worth saying: the
+    // v2 reader now reports frames it declined, so replacing rather than joining
+    // would drop them the moment the file also carries a v1 block — which is most
+    // of the files this applies to.
+    return {
+        ...tag,
+        fallbackTags: older.tags,
+        refusals: recover([...(tag.refusals ?? []), ...older.refusals]),
+        encoding,
+    };
+}
+/** Where the ID3v2 tag ends and the audio begins. */
+function id3End(bytes) {
+    const size = (((bytes[6] ?? 0) & 0x7f) << 21) |
+        (((bytes[7] ?? 0) & 0x7f) << 14) |
+        (((bytes[8] ?? 0) & 0x7f) << 7) |
+        ((bytes[9] ?? 0) & 0x7f);
+    // A footer, when present, is another ten bytes after the tag — and it sits
+    // *before* the audio, so not counting it starts the search ten bytes early.
+    const footer = (((bytes[5] ?? 0) & 0x10) !== 0 ? 10 : 0);
+    return Math.min(10 + size + footer, bytes.length);
+}

package/dist/tags/types.js

@@ -0,0 +1,51 @@
+/** The picture type that means "this is the cover", in every format here. */
+export const FRONT_COVER = 3;
+/**
+ * The better of two pictures the same file holds.
+ *
+ * A file may carry several — front, back, artist, a photograph of the disc — and
+ * the cover is the one to keep. A front cover wins over anything; between two of
+ * the same standing the first one seen wins, so that the answer is a function of
+ * the file's bytes rather than of the order a walk happened to visit boxes in.
+ */
+export function betterPicture(current, candidate) {
+    if (current === undefined)
+        return candidate;
+    if (current.kind === FRONT_COVER)
+        return current;
+    return candidate.kind === FRONT_COVER ? candidate : current;
+}
+/**
+ * A fresh empty answer, as a function and not a shared constant.
+ *
+ * A constant would hand every caller the same `tags` array, and the first
+ * reader to push into it would be filling in the answer for every later file.
+ */
+export function noTags(container = null, codec = null) {
+    return {
+        tags: [],
+        container,
+        codec,
+        durationMs: null,
+        durationRefused: false,
+        sampleRate: null,
+        channels: null,
+        bitsPerSample: null,
+        encoding: null,
+    };
+}
+/**
+ * Keep the less certain of two decode verdicts.
+ *
+ * Ties keep the one already held, which makes the answer a function of the
+ * file's bytes and their order rather than of anything ambient — the same
+ * unchanged file must not report a different encoding on two machines.
+ */
+export function weakestEncoding(current, candidate) {
+    const verdict = {
+        encoding: candidate.encoding,
+        confidence: candidate.confidence,
+        basis: candidate.basis,
+    };
+    return current === null || verdict.confidence < current.confidence ? verdict : current;
+}

package/dist/tags/vorbis-comment.js

@@ -0,0 +1,155 @@
+import { decodeVorbisText, looksLikeText } from "../text/encoding.js";
+import { pictureBlockAt } from "./picture.js";
+import { FRONT_COVER, betterPicture, noTags, weakestEncoding } from "./types.js";
+/**
+ * The comment name that carries a picture, and the reason Ogg files have covers
+ * at all: the container has no metadata block to put one in, so the Vorbis
+ * convention is to base64 a FLAC picture block (RFC 9639 §8.8) into a comment.
+ *
+ * Read as a comment and it looks like any other tag, which is the trap: taken at
+ * face value it becomes a tag row of a hundred and eighty kilobytes of base64,
+ * out of a meta layer that exists to describe a collection rather than to carry
+ * one. Measured before this was written: eighty files put nineteen and a half
+ * megabytes into `file_tag`, one of them in a single value of 562 KB.
+ *
+ * The older `COVERART`/`COVERARTMIME` pair does the same job, in two comments
+ * with the image base64'd raw and no block around it. No file in this collection
+ * uses it, and it is not read here — but it is the reason the name below is
+ * spelled out in full rather than matched loosely.
+ */
+const PICTURE_COMMENT = 'metadata_block_picture';
+/**
+ * The better of two pictures a comment list carries, by the rule `betterPicture`
+ * applies to the ones a file carries as ranges.
+ *
+ * A front cover wins over anything; between two of the same standing the first
+ * one seen wins, so that the answer is a function of the file's bytes rather
+ * than of the order a walk happened to visit them in.
+ */
+function betterCommentPicture(current, candidate) {
+    if (current === null)
+        return candidate;
+    if (current.kind === FRONT_COVER)
+        return current;
+    return candidate.kind === FRONT_COVER ? candidate : current;
+}
+/**
+ * The picture one `METADATA_BLOCK_PICTURE` value holds, or null.
+ *
+ * Base64 is decoded leniently by Node — it skips characters it does not know
+ * rather than refusing — so a value that is not base64 at all decodes to
+ * something rather than throwing, and the block parser above is what decides
+ * whether the result is a picture. A value that does not parse is dropped
+ * silently, which is right for this one name and wrong for every other: an
+ * unreadable *picture* is a file with no cover, while an unreadable tag is a
+ * finding about the file that the caller counts and reports.
+ */
+function pictureFromComment(value) {
+    const decoded = Buffer.from(Buffer.from(value).toString('latin1'), 'base64');
+    const block = pictureBlockAt(decoded, 0, decoded.length);
+    if (block === null)
+        return null;
+    return {
+        mime: block.mime,
+        kind: block.kind,
+        data: decoded.subarray(block.dataAt, block.dataAt + block.dataLength),
+    };
+}
+/**
+ * Read a comment list occupying `[at, end)`, and report the picture it carried.
+ *
+ * Two things about this are worth knowing before the code below reads as
+ * arbitrary. The lengths are 32-bit little-endian, not the big-endian a reader
+ * arriving from FLAC's *blocks* would expect — §5.2.1 says the fields are packed
+ * "lsb first" and, being octet-aligned, "can simply be read as unaligned 32 bit
+ * little endian unsigned integers". And the field *contents* are UTF-8 while the
+ * field *names* are ASCII, so a name in any other script is something a file can
+ * carry and the specification forbids at once.
+ *
+ * The walk stops at the first entry it cannot believe — a length that runs past
+ * the end, a count that promises more than is there — and keeps what it already
+ * read, because the caller is a scan over a whole collection and one damaged
+ * file must not be able to stop it.
+ */
+export function readCommentList(bytes, at, end, into) {
+    let cursor = at;
+    let picture = null;
+    const takeLength = () => {
+        if (cursor + 4 > end)
+            return null;
+        const value = ((bytes[cursor] ?? 0) |
+            ((bytes[cursor + 1] ?? 0) << 8) |
+            ((bytes[cursor + 2] ?? 0) << 16) |
+            ((bytes[cursor + 3] ?? 0) << 24)) >>>
+            0;
+        cursor += 4;
+        return value;
+    };
+    const vendorLength = takeLength();
+    if (vendorLength === null)
+        return null;
+    // The vendor string names the encoder. Nothing here reads it, but its length
+    // is exactly what stands between this cursor and the first comment.
+    cursor += vendorLength;
+    if (cursor > end)
+        return null;
+    const count = takeLength();
+    if (count === null)
+        return null;
+    for (let i = 0; i < count; i += 1) {
+        const size = takeLength();
+        if (size === null || cursor + size > end)
+            return picture;
+        const entry = bytes.subarray(cursor, cursor + size);
+        cursor += size;
+        const equals = entry.indexOf(0x3d); // '='
+        // §5.2.2 defines a field as a name, an `=`, and the contents. An entry with
+        // no separator, or with one in first place, is not a field with a missing
+        // half — it never named anything, so there is no value being lost. This is
+        // the only shape dropped outright, and the only shape that can be.
+        if (equals <= 0)
+            continue;
+        // §5.2.2: names are case-insensitive, so they are folded on the way in and
+        // every later stage compares one way. Enforcing the section's other rule —
+        // a name holds only U+0020..U+007D, which RFC 9639 §8.6 widens to
+        // U+0020..U+007E without `=` for the FLAC comments this reader also serves —
+        // would mean dropping a field or renaming it, both of which lose what the file
+        // said. A name nothing downstream matches is inert; a value thrown away is not.
+        const named = decodeVorbisText(entry.subarray(0, equals));
+        into.encoding = weakestEncoding(into.encoding, named);
+        const name = named.text.toLowerCase();
+        // The picture is taken out here, before anything is done to the value: it is
+        // not a tag, and the decoding below would turn it into one — a hundred and
+        // eighty kilobytes of base64 in a table of names.
+        if (name === PICTURE_COMMENT) {
+            const found = pictureFromComment(entry.subarray(equals + 1));
+            if (found !== null)
+                picture = betterCommentPicture(picture, found);
+            continue;
+        }
+        const decoded = decodeVorbisText(entry.subarray(equals + 1));
+        // A value that is not valid UTF-8 still comes out as something readable —
+        // it just may be readable and wrong. Recording how it was decided is what
+        // lets a later stage say so instead of printing mojibake with confidence.
+        // The verdict is kept whether or not the value itself survives; a comment
+        // that turned out not to be text is still a finding about the file.
+        into.encoding = weakestEncoding(into.encoding, decoded);
+        if (!looksLikeText(decoded.text))
+            continue;
+        into.tags.push({ name, value: decoded.text });
+    }
+    return picture;
+}
+/**
+ * The picture a comment *packet* carries, and nothing else.
+ *
+ * The serving side of `readCommentList`, and deliberately the same code: a cover
+ * is fetched long after the scan that noticed it, from a file nothing is holding
+ * in memory, so the bytes have to be derived again — and the derivation has to
+ * be the one the scan used, or the two can disagree about which comment is the
+ * picture. The names are read into a throwaway answer rather than a second
+ * parser.
+ */
+export function pictureInComment(bytes, at, end) {
+    return readCommentList(bytes, at, end, noTags());
+}