@wavehouse/chtypes 0.1.0

package/dist/tar.js

@@ -0,0 +1,310 @@
+/**
+ * A streaming reader for the artifact tarballs — ustar, pax extended
+ * headers, GNU long names — written here rather than taken from a
+ * dependency: this package's one runtime dependency is ffi-rs, and a reader
+ * that extracts regular files under one directory is a page of code.
+ *
+ * Files only. Directories are created; everything else — symlinks, hard
+ * links, devices, FIFOs — is skipped without being followed: an artifact is
+ * `manifest.json`, one library and two text files, and a link inside one is
+ * an attack, not a feature. Every entry name is confined to the destination
+ * (no absolute names, no `..` components, no escape once resolved), and a
+ * name that escapes is refused as corrupt rather than clamped.
+ *
+ * The archive is consumed as a stream: a 300 MB library is written to disk
+ * as it inflates and never sits in memory whole.
+ */
+import { createReadStream } from 'node:fs';
+import { mkdir, open as openFile } from 'node:fs/promises';
+import path from 'node:path';
+import { Writable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import { createGunzip } from 'node:zlib';
+import { ArtifactCorruptError, FetchError } from './errors.js';
+const BLOCK = 512;
+/** pax records and long names are tiny; a megabyte of them is not a tarball. */
+const MAX_META = 1 << 20;
+/**
+ * Inflate and unpack `archive` (a `.tar.gz`) into `dest`, which must already
+ * exist. Returns the regular files written, in archive order.
+ *
+ * @throws {ArtifactCorruptError} when the bytes are not a gzip stream, a
+ *   header fails its checksum, the archive is truncated, or an entry name
+ *   would land outside `dest`.
+ */
+export async function extractTarGz(archive, dest) {
+    const extractor = new TarExtractor(path.resolve(dest));
+    try {
+        await pipeline(createReadStream(archive), createGunzip(), extractor);
+    }
+    catch (err) {
+        if (err instanceof FetchError)
+            throw err;
+        throw new ArtifactCorruptError(`chtypes: ${archive} is not a readable tar.gz: ${errorText(err)}`, {
+            cause: err,
+        });
+    }
+    return extractor.entries;
+}
+class TarExtractor extends Writable {
+    dest;
+    entries = [];
+    head = Buffer.alloc(BLOCK);
+    headFill = 0;
+    /** Data bytes still to consume for the current entry, then the block padding after them. */
+    remaining = 0;
+    padding = 0;
+    pending = null;
+    /** Overrides a pax `x` or GNU `L` entry set for the entry that follows. */
+    nextName = null;
+    nextSize = null;
+    zeroBlocks = 0;
+    ended = false;
+    constructor(dest) {
+        super();
+        this.dest = dest;
+    }
+    _write(chunk, _encoding, callback) {
+        this.consume(chunk).then(() => callback(), (err) => callback(err instanceof Error ? err : new Error(String(err))));
+    }
+    _final(callback) {
+        const truncated = !this.ended && (this.headFill !== 0 || this.remaining !== 0 || this.pending !== null);
+        this.closePending().then(() => callback(truncated ? new ArtifactCorruptError('chtypes: tarball is truncated') : null), (err) => callback(err instanceof Error ? err : new Error(String(err))));
+    }
+    _destroy(error, callback) {
+        this.closePending().then(() => callback(error), () => callback(error));
+    }
+    async closePending() {
+        const p = this.pending;
+        this.pending = null;
+        if (p !== null && p.kind === 'file')
+            await p.handle.close();
+    }
+    async consume(chunk) {
+        let off = 0;
+        while (off < chunk.length) {
+            // Bytes after the end-of-archive marker are padding; nothing lives there.
+            if (this.ended)
+                return;
+            if (this.remaining > 0) {
+                const n = Math.min(this.remaining, chunk.length - off);
+                await this.data(chunk.subarray(off, off + n));
+                this.remaining -= n;
+                off += n;
+                if (this.remaining === 0)
+                    await this.finishEntry();
+                continue;
+            }
+            if (this.padding > 0) {
+                const n = Math.min(this.padding, chunk.length - off);
+                this.padding -= n;
+                off += n;
+                continue;
+            }
+            const n = Math.min(BLOCK - this.headFill, chunk.length - off);
+            chunk.copy(this.head, this.headFill, off, off + n);
+            this.headFill += n;
+            off += n;
+            if (this.headFill === BLOCK) {
+                this.headFill = 0;
+                await this.header(this.head);
+            }
+        }
+    }
+    async header(block) {
+        if (isZero(block)) {
+            // Two zero blocks end the archive; one followed by EOF is tolerated in _final.
+            this.zeroBlocks += 1;
+            if (this.zeroBlocks >= 2)
+                this.ended = true;
+            return;
+        }
+        this.zeroBlocks = 0;
+        if (!checksumOk(block))
+            throw new ArtifactCorruptError('chtypes: tarball header fails its checksum');
+        const flag = block[156];
+        const ownSize = parseSize(block, 124, 12);
+        if (Number.isNaN(ownSize))
+            throw new ArtifactCorruptError('chtypes: tarball header carries an unreadable size');
+        const isMeta = flag === 0x78 /* x */ || flag === 0x67 /* g */ || flag === 0x4c /* L */;
+        let name;
+        let size;
+        if (isMeta) {
+            name = '';
+            size = ownSize;
+            if (size > MAX_META)
+                throw new ArtifactCorruptError('chtypes: tarball extended header is implausibly large');
+            this.pending = { kind: 'meta', flag, chunks: [] };
+        }
+        else {
+            const magic = block.toString('latin1', 257, 262);
+            const prefix = magic === 'ustar' ? cstr(block, 345, 155) : '';
+            const base = cstr(block, 0, 100);
+            name = this.nextName ?? (prefix !== '' ? `${prefix}/${base}` : base);
+            size = this.nextSize ?? ownSize;
+            this.nextName = null;
+            this.nextSize = null;
+            if (flag === 0x30 /* 0 */ || flag === 0x00 || flag === 0x37 /* 7 */) {
+                const target = this.confine(name, false);
+                await mkdir(path.dirname(target), { recursive: true });
+                const handle = await openFile(target, 'w');
+                this.pending = { kind: 'file', handle };
+                this.entries.push({ name: path.relative(this.dest, target).split(path.sep).join('/'), size });
+            }
+            else if (flag === 0x35 /* 5 */) {
+                const target = this.confine(name, true);
+                if (target !== this.dest)
+                    await mkdir(target, { recursive: true });
+                this.pending = { kind: 'skip' };
+            }
+            else {
+                // Links, devices, FIFOs: files only. Skipped, never followed.
+                this.pending = { kind: 'skip' };
+            }
+        }
+        this.remaining = size;
+        this.padding = (BLOCK - (size % BLOCK)) % BLOCK;
+        if (size === 0)
+            await this.finishEntry();
+    }
+    async data(slice) {
+        const p = this.pending;
+        if (p === null)
+            return;
+        if (p.kind === 'file') {
+            let written = 0;
+            while (written < slice.length) {
+                const r = await p.handle.write(slice, written, slice.length - written);
+                if (r.bytesWritten <= 0)
+                    throw new Error('short write');
+                written += r.bytesWritten;
+            }
+        }
+        else if (p.kind === 'meta') {
+            p.chunks.push(Buffer.from(slice));
+        }
+    }
+    async finishEntry() {
+        const p = this.pending;
+        this.pending = null;
+        if (p === null)
+            return;
+        if (p.kind === 'file') {
+            await p.handle.close();
+        }
+        else if (p.kind === 'meta') {
+            const payload = Buffer.concat(p.chunks);
+            if (p.flag === 0x4c /* L: GNU long name */) {
+                this.nextName = cstr(payload, 0, payload.length);
+            }
+            else if (p.flag === 0x78 /* x: pax, for the next entry */) {
+                for (const [key, value] of paxRecords(payload)) {
+                    if (key === 'path')
+                        this.nextName = value;
+                    else if (key === 'size') {
+                        const n = Number(value);
+                        if (!/^\d+$/.test(value) || !Number.isSafeInteger(n)) {
+                            throw new ArtifactCorruptError('chtypes: tarball pax header carries an unreadable size');
+                        }
+                        this.nextSize = n;
+                    }
+                }
+            }
+            // 'g' (global pax) carries nothing an artifact needs; consumed, ignored.
+        }
+    }
+    /** Resolve an entry name under the destination, refusing anything that would leave it. */
+    confine(name, isDir) {
+        const bad = (why) => {
+            throw new ArtifactCorruptError(`chtypes: tarball entry ${JSON.stringify(name)} refused: ${why}`);
+        };
+        if (name === '')
+            bad('empty name');
+        if (name.includes('\\'))
+            bad('backslash in name');
+        if (name.startsWith('/'))
+            bad('absolute path');
+        const parts = name.split('/').filter((p) => p !== '' && p !== '.');
+        if (parts.some((p) => p === '..'))
+            bad('path traversal');
+        if (parts.length === 0) {
+            if (isDir)
+                return this.dest;
+            bad('no file name');
+        }
+        const target = path.join(this.dest, ...parts);
+        if (target !== this.dest && !target.startsWith(this.dest + path.sep))
+            bad('escapes the destination');
+        return target;
+    }
+}
+function isZero(block) {
+    for (let i = 0; i < block.length; i++)
+        if (block[i] !== 0)
+            return false;
+    return true;
+}
+/** The header checksum, with the checksum field itself read as spaces; unsigned and signed sums both accepted. */
+function checksumOk(block) {
+    const recorded = parseOctal(block, 148, 8);
+    if (Number.isNaN(recorded))
+        return false;
+    let unsigned = 0;
+    let signed = 0;
+    for (let i = 0; i < BLOCK; i++) {
+        const b = i >= 148 && i < 156 ? 0x20 : block[i];
+        unsigned += b;
+        signed += b >= 0x80 ? b - 0x100 : b;
+    }
+    return recorded === unsigned || recorded === signed;
+}
+function cstr(buf, off, len) {
+    const end = Math.min(buf.length, off + len);
+    let nul = off;
+    while (nul < end && buf[nul] !== 0)
+        nul++;
+    return buf.toString('utf8', off, nul);
+}
+function parseOctal(buf, off, len) {
+    const text = cstr(buf, off, len).trim();
+    if (text === '')
+        return 0;
+    if (!/^[0-7]+$/.test(text))
+        return Number.NaN;
+    return Number.parseInt(text, 8);
+}
+/** Sizes are octal, or base-256 with the high bit of the first byte set (GNU, for > 8 GiB). */
+function parseSize(buf, off, len) {
+    const first = buf[off];
+    if ((first & 0x80) !== 0) {
+        let value = first & 0x7f;
+        for (let i = 1; i < len; i++)
+            value = value * 256 + buf[off + i];
+        return Number.isSafeInteger(value) ? value : Number.NaN;
+    }
+    return parseOctal(buf, off, len);
+}
+/** pax records: `<decimal length> <key>=<value>\n`, the length counting the whole record. */
+function paxRecords(payload) {
+    const out = [];
+    let pos = 0;
+    while (pos < payload.length) {
+        const space = payload.indexOf(0x20, pos);
+        if (space < 0)
+            break;
+        const len = Number.parseInt(payload.toString('latin1', pos, space), 10);
+        if (!Number.isInteger(len) || len <= 0 || pos + len > payload.length) {
+            throw new ArtifactCorruptError('chtypes: tarball pax header is malformed');
+        }
+        const record = payload.toString('utf8', space + 1, pos + len - 1);
+        const eq = record.indexOf('=');
+        if (eq > 0)
+            out.push([record.slice(0, eq), record.slice(eq + 1)]);
+        pos += len;
+    }
+    return out;
+}
+function errorText(err) {
+    return err instanceof Error ? err.message : String(err);
+}
+//# sourceMappingURL=tar.js.map

package/dist/tar.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tar.js","sourceRoot":"","sources":["../src/tar.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,KAAK,EAAE,IAAI,IAAI,QAAQ,EAAmB,MAAM,kBAAkB,CAAC;AAC5E,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE/D,MAAM,KAAK,GAAG,GAAG,CAAC;AAClB,gFAAgF;AAChF,MAAM,QAAQ,GAAG,CAAC,IAAI,EAAE,CAAC;AASzB;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAAe,EAAE,IAAY;IAC9D,MAAM,SAAS,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;IACvD,IAAI,CAAC;QACH,MAAM,QAAQ,CAAC,gBAAgB,CAAC,OAAO,CAAC,EAAE,YAAY,EAAE,EAAE,SAAS,CAAC,CAAC;IACvE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,YAAY,UAAU;YAAE,MAAM,GAAG,CAAC;QACzC,MAAM,IAAI,oBAAoB,CAAC,YAAY,OAAO,8BAA8B,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE;YAChG,KAAK,EAAE,GAAG;SACX,CAAC,CAAC;IACL,CAAC;IACD,OAAO,SAAS,CAAC,OAAO,CAAC;AAC3B,CAAC;AAOD,MAAM,YAAa,SAAQ,QAAQ;IAeJ,IAAI;IAdxB,OAAO,GAAqB,EAAE,CAAC;IAEvB,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACpC,QAAQ,GAAG,CAAC,CAAC;IACrB,4FAA4F;IACpF,SAAS,GAAG,CAAC,CAAC;IACd,OAAO,GAAG,CAAC,CAAC;IACZ,OAAO,GAAmB,IAAI,CAAC;IACvC,2EAA2E;IACnE,QAAQ,GAAkB,IAAI,CAAC;IAC/B,QAAQ,GAAkB,IAAI,CAAC;IAC/B,UAAU,GAAG,CAAC,CAAC;IACf,KAAK,GAAG,KAAK,CAAC;IAEtB,YAA6B,IAAY;QACvC,KAAK,EAAE,CAAC;oBADmB,IAAI;IAEjC,CAAC;IAEQ,MAAM,CAAC,KAAa,EAAE,SAAyB,EAAE,QAAwC;QAChG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CACtB,GAAG,EAAE,CAAC,QAAQ,EAAE,EAChB,CAAC,GAAY,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAChF,CAAC;IACJ,CAAC;IAEQ,MAAM,CAAC,QAAwC;QACtD,MAAM,SAAS,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,QAAQ,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS,KAAK,CAAC,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC;QACxG,IAAI,CAAC,YAAY,EAAE,CAAC,IAAI,CACtB,GAAG,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,oBAAoB,CAAC,+BAA+B,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAC5F,CAAC,GAAY,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAChF,CAAC;IACJ,CAAC;IAEQ,QAAQ,CAAC,KAAmB,EAAE,QAAwC;QAC7E,IAAI,CAAC,YAAY,EAAE,CAAC,IAAI,CACtB,GAAG,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EACrB,GAAG,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,CACtB,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC;QACvB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM;YAAE,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAC9D,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,KAAa;QACjC,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,OAAO,GAAG,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC1B,0EAA0E;YAC1E,IAAI,IAAI,CAAC,KAAK;gBAAE,OAAO;YACvB,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;gBACvD,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;gBAC9C,IAAI,CAAC,SAAS,IAAI,CAAC,CAAC;gBACpB,GAAG,IAAI,CAAC,CAAC;gBACT,IAAI,IAAI,CAAC,SAAS,KAAK,CAAC;oBAAE,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;gBACnD,SAAS;YACX,CAAC;YACD,IAAI,IAAI,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;gBACrB,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;gBACrD,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;gBAClB,GAAG,IAAI,CAAC,CAAC;gBACT,SAAS;YACX,CAAC;YACD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;YAC9D,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,EAAE,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;YACnD,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;YACnB,GAAG,IAAI,CAAC,CAAC;YACT,IAAI,IAAI,CAAC,QAAQ,KAAK,KAAK,EAAE,CAAC;gBAC5B,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;gBAClB,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,MAAM,CAAC,KAAa;QAChC,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAClB,+EAA+E;YAC/E,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;YACrB,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC;gBAAE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;YAC5C,OAAO;QACT,CAAC;QACD,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC;QACpB,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;YAAE,MAAM,IAAI,oBAAoB,CAAC,4CAA4C,CAAC,CAAC;QAErG,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAE,CAAC;QACzB,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QAC1C,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC;YAAE,MAAM,IAAI,oBAAoB,CAAC,oDAAoD,CAAC,CAAC;QAChH,MAAM,MAAM,GAAG,IAAI,KAAK,IAAI,CAAC,OAAO,IAAI,IAAI,KAAK,IAAI,CAAC,OAAO,IAAI,IAAI,KAAK,IAAI,CAAC,OAAO,CAAC;QAEvF,IAAI,IAAY,CAAC;QACjB,IAAI,IAAY,CAAC;QACjB,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,GAAG,EAAE,CAAC;YACV,IAAI,GAAG,OAAO,CAAC;YACf,IAAI,IAAI,GAAG,QAAQ;gBAAE,MAAM,IAAI,oBAAoB,CAAC,uDAAuD,CAAC,CAAC;YAC7G,IAAI,CAAC,OAAO,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;QACpD,CAAC;aAAM,CAAC;YACN,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;YACjD,MAAM,MAAM,GAAG,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;YACjC,IAAI,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACrE,IAAI,GAAG,IAAI,CAAC,QAAQ,IAAI,OAAO,CAAC;YAChC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;YACrB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;YAErB,IAAI,IAAI,KAAK,IAAI,CAAC,OAAO,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,CAAC,OAAO,EAAE,CAAC;gBACpE,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBACzC,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBACvD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAC3C,IAAI,CAAC,OAAO,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;gBACxC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;YAChG,CAAC;iBAAM,IAAI,IAAI,KAAK,IAAI,CAAC,OAAO,EAAE,CAAC;gBACjC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBACxC,IAAI,MAAM,KAAK,IAAI,CAAC,IAAI;oBAAE,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBACnE,IAAI,CAAC,OAAO,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACN,8DAA8D;gBAC9D,IAAI,CAAC,OAAO,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;YAClC,CAAC;QACH,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,OAAO,GAAG,CAAC,KAAK,GAAG,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC;QAChD,IAAI,IAAI,KAAK,CAAC;YAAE,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;IAC3C,CAAC;IAEO,KAAK,CAAC,IAAI,CAAC,KAAa;QAC9B,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC;QACvB,IAAI,CAAC,KAAK,IAAI;YAAE,OAAO;QACvB,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACtB,IAAI,OAAO,GAAG,CAAC,CAAC;YAChB,OAAO,OAAO,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBAC9B,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC;gBACvE,IAAI,CAAC,CAAC,YAAY,IAAI,CAAC;oBAAE,MAAM,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC;gBACxD,OAAO,IAAI,CAAC,CAAC,YAAY,CAAC;YAC5B,CAAC;QACH,CAAC;aAAM,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC7B,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,WAAW;QACvB,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC;QACvB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,KAAK,IAAI;YAAE,OAAO;QACvB,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACtB,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACzB,CAAC;aAAM,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC7B,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACxC,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,sBAAsB,EAAE,CAAC;gBAC3C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YACnD,CAAC;iBAAM,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,gCAAgC,EAAE,CAAC;gBAC5D,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC/C,IAAI,GAAG,KAAK,MAAM;wBAAE,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;yBACrC,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;wBACxB,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;wBACxB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;4BACrD,MAAM,IAAI,oBAAoB,CAAC,wDAAwD,CAAC,CAAC;wBAC3F,CAAC;wBACD,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;oBACpB,CAAC;gBACH,CAAC;YACH,CAAC;YACD,yEAAyE;QAC3E,CAAC;IACH,CAAC;IAED,0FAA0F;IAClF,OAAO,CAAC,IAAY,EAAE,KAAc;QAC1C,MAAM,GAAG,GAAG,CAAC,GAAW,EAAS,EAAE;YACjC,MAAM,IAAI,oBAAoB,CAAC,0BAA0B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,CAAC;QACnG,CAAC,CAAC;QACF,IAAI,IAAI,KAAK,EAAE;YAAE,GAAG,CAAC,YAAY,CAAC,CAAC;QACnC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,GAAG,CAAC,mBAAmB,CAAC,CAAC;QAClD,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,GAAG,CAAC,eAAe,CAAC,CAAC;QAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;QACnE,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC;YAAE,GAAG,CAAC,gBAAgB,CAAC,CAAC;QACzD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,IAAI,KAAK;gBAAE,OAAO,IAAI,CAAC,IAAI,CAAC;YAC5B,GAAG,CAAC,cAAc,CAAC,CAAC;QACtB,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,KAAK,CAAC,CAAC;QAC9C,IAAI,MAAM,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC;YAAE,GAAG,CAAC,yBAAyB,CAAC,CAAC;QACrG,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAED,SAAS,MAAM,CAAC,KAAa;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE;QAAE,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;IACxE,OAAO,IAAI,CAAC;AACd,CAAC;AAED,kHAAkH;AAClH,SAAS,UAAU,CAAC,KAAa;IAC/B,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;IAC3C,IAAI,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC;QAAE,OAAO,KAAK,CAAC;IACzC,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,MAAM,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;QACjD,QAAQ,IAAI,CAAC,CAAC;QACd,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,MAAM,CAAC;AACtD,CAAC;AAED,SAAS,IAAI,CAAC,GAAW,EAAE,GAAW,EAAE,GAAW;IACjD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;IAC5C,IAAI,GAAG,GAAG,GAAG,CAAC;IACd,OAAO,GAAG,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC;QAAE,GAAG,EAAE,CAAC;IAC1C,OAAO,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;AACxC,CAAC;AAED,SAAS,UAAU,CAAC,GAAW,EAAE,GAAW,EAAE,GAAW;IACvD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACxC,IAAI,IAAI,KAAK,EAAE;QAAE,OAAO,CAAC,CAAC;IAC1B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,MAAM,CAAC,GAAG,CAAC;IAC9C,OAAO,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;AAClC,CAAC;AAED,+FAA+F;AAC/F,SAAS,SAAS,CAAC,GAAW,EAAE,GAAW,EAAE,GAAW;IACtD,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAE,CAAC;IACxB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,IAAI,KAAK,GAAG,KAAK,GAAG,IAAI,CAAC;QACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE;YAAE,KAAK,GAAG,KAAK,GAAG,GAAG,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,CAAE,CAAC;QAClE,OAAO,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;IAC1D,CAAC;IACD,OAAO,UAAU,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;AACnC,CAAC;AAED,6FAA6F;AAC7F,SAAS,UAAU,CAAC,OAAe;IACjC,MAAM,GAAG,GAA4B,EAAE,CAAC;IACxC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,OAAO,GAAG,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACzC,IAAI,KAAK,GAAG,CAAC;YAAE,MAAM;QACrB,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;QACxE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,GAAG,GAAG,GAAG,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;YACrE,MAAM,IAAI,oBAAoB,CAAC,0CAA0C,CAAC,CAAC;QAC7E,CAAC;QACD,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,EAAE,KAAK,GAAG,CAAC,EAAE,GAAG,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC;QAClE,MAAM,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,EAAE,GAAG,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAClE,GAAG,IAAI,GAAG,CAAC;IACb,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,SAAS,CAAC,GAAY;IAC7B,OAAO,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;AAC1D,CAAC"}

package/dist/transform.d.ts

@@ -0,0 +1,73 @@
+/**
+ * Silent-transformation detection — a port of `go/chtypes/transform.go`.
+ *
+ * ClickHouse has no notion of "I changed your value": `readIntText` wraps mod
+ * 2^N, `SerializationDateTime` truncates, `parseUUID` maps every non-hex byte to
+ * 0xff, and all of it returns success. So the fact of a change has to be
+ * established from outside, and this is the one part of chtypes that is derived
+ * rather than executed by ClickHouse — which makes it the one part a binding can
+ * get wrong without the C library noticing. Omitting it scores 0 % recall on the
+ * arbiter's transformation axis regardless of everything else.
+ *
+ * Two independent detectors run and their union is reported:
+ *
+ *  1. **Supplied vs stored** — the raw input text against ClickHouse's rendering
+ *     of what it kept. Models nothing; catches changes a widened type makes
+ *     identically (2024-02-30 -> 2024-03-01) and types with no wider type to
+ *     compare against (Float64, Int256, String).
+ *  2. **Reference type** — the C layer already parsed each field a second time
+ *     through a structurally identical type with widened leaves and reported it
+ *     as `ref` / `ref_type`. This names the reason precisely (an overflow wrap
+ *     versus a decimal truncation) and still fires where the supplied text is
+ *     not comparable (a CSV field, a base64 blob).
+ *
+ * Neither reimplements a coercion rule: the first compares two values already in
+ * hand, the second compares two ClickHouse answers.
+ *
+ * Both compare **bytes**. A supplied field and a stored `String` can each be any
+ * byte sequence, and two different byte sequences that both decode to U+FFFD are
+ * not the same value — deciding on decoded text would silently merge them, and
+ * `equivalent`'s fast path would then call a real change identical.
+ */
+import type { ColumnDoc, Transform } from './results.js';
+/** The stable reason vocabulary. The harness groups on these strings. */
+export declare const Reason: {
+    readonly OverflowWrap: 'overflow_wrap';
+    readonly NullToDefault: 'null_to_default';
+    readonly NullLoss: 'null_loss';
+    readonly DecimalTruncate: 'decimal_truncate';
+    readonly DateClamp: 'date_clamp';
+    readonly DateTimeWrap: 'datetime_wrap';
+    readonly DateShift: 'date_shift';
+    readonly UuidMangle: 'uuid_mangle';
+    readonly IpMangle: 'ip_mangle';
+    readonly FloatPrecision: 'float_precision';
+    readonly LossyNumeric: 'lossy_numeric';
+    readonly FixedStringPad: 'fixedstring_pad';
+    readonly Emptied: 'emptied';
+    readonly ElementChanged: 'element_changed';
+    readonly EnumCoerce: 'enum_coerce';
+    readonly ValueChanged: 'value_changed';
+    readonly Poisoned: 'poisoned';
+    readonly DuplicateKeyDropped: 'duplicate_key_dropped';
+    readonly Reformat: 'reformat';
+    readonly DefaultFilled: 'default_filled';
+    readonly ZeroFilled: 'zero_filled';
+    readonly DefaultMaterialised: 'default_materialised';
+    readonly TtlExpired: 'ttl_expired';
+    readonly TtlColumnExpired: 'ttl_column_expired';
+};
+/** One of the stable reason strings — see the `Reason` constant object. */
+export type Reason = (typeof Reason)[keyof typeof Reason];
+/**
+ * Whether a transform reason means information was LOST (a warning to raise to
+ * the tenant) rather than merely re-spelled or filled in. False for exactly
+ * four reasons — `reformat`, `default_filled`, `zero_filled`,
+ * `default_materialised` — true for everything else, unknown strings included:
+ * a reason nobody has audited must degrade to a warning, never to silence.
+ * Mirrors `Transform#lossy`. Never throws.
+ */
+export declare function isLossyReason(reason: string): boolean;
+/** Classify one column of a row document. */
+export declare function classify(c: ColumnDoc): Transform[];
+//# sourceMappingURL=transform.d.ts.map

package/dist/transform.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"transform.d.ts","sourceRoot":"","sources":["../src/transform.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAIH,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzD,yEAAyE;AACzE,eAAO,MAAM,MAAM;aACjB,YAAY,EAAE,eAAe;aAC7B,aAAa,EAAE,iBAAiB;aAChC,QAAQ,EAAE,WAAW;aACrB,eAAe,EAAE,kBAAkB;aACnC,SAAS,EAAE,YAAY;aACvB,YAAY,EAAE,eAAe;aAC7B,SAAS,EAAE,YAAY;aACvB,UAAU,EAAE,aAAa;aACzB,QAAQ,EAAE,WAAW;aACrB,cAAc,EAAE,iBAAiB;aACjC,YAAY,EAAE,eAAe;aAC7B,cAAc,EAAE,iBAAiB;aACjC,OAAO,EAAE,SAAS;aAClB,cAAc,EAAE,iBAAiB;aACjC,UAAU,EAAE,aAAa;aACzB,YAAY,EAAE,eAAe;aAC7B,QAAQ,EAAE,UAAU;aACpB,mBAAmB,EAAE,uBAAuB;aAC5C,QAAQ,EAAE,UAAU;aACpB,aAAa,EAAE,gBAAgB;aAC/B,UAAU,EAAE,aAAa;aACzB,mBAAmB,EAAE,sBAAsB;aAC3C,UAAU,EAAE,aAAa;aACzB,gBAAgB,EAAE,oBAAoB;CAC9B,CAAC;AAEX,2EAA2E;AAC3E,MAAM,MAAM,MAAM,GAAG,CAAC,OAAO,MAAM,CAAC,CAAC,MAAM,OAAO,MAAM,CAAC,CAAC;AAe1D;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAErD;AAsBD,6CAA6C;AAC7C,wBAAgB,QAAQ,CAAC,CAAC,EAAE,SAAS,GAAG,SAAS,EAAE,CAkFlD"}