@abyss-project/main 34.1.2 → 34.2.0

package/dist/server/archive/archive-seal.util.d.ts

@@ -0,0 +1,7 @@
+export declare const ARCHIVE_KEY_BYTES = 32;
+export declare const seal: (plaintext: Buffer, key: Buffer) => Buffer;
+export declare const unseal: (payload: Buffer, key: Buffer) => Buffer;
+export declare const generateArchiveKey: () => Buffer;
+export declare const parseMasterKey: (raw: string, variableName: string) => Buffer;
+export declare const wrapKey: (organizationKey: Buffer, masterKey: Buffer) => string;
+export declare const unwrapKey: (wrappedKey: string, masterKey: Buffer) => Buffer;

package/dist/server/archive/archive-seal.util.js

@@ -0,0 +1,75 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.unwrapKey = exports.wrapKey = exports.parseMasterKey = exports.generateArchiveKey = exports.unseal = exports.seal = exports.ARCHIVE_KEY_BYTES = void 0;
+const crypto = __importStar(require("crypto"));
+const ALGORITHM = 'aes-256-gcm';
+exports.ARCHIVE_KEY_BYTES = 32;
+const IV_BYTES = 12;
+const AUTH_TAG_BYTES = 16;
+const seal = (plaintext, key) => {
+    const iv = crypto.randomBytes(IV_BYTES);
+    const cipher = crypto.createCipheriv(ALGORITHM, key, iv);
+    const ciphertext = Buffer.concat([cipher.update(plaintext), cipher.final()]);
+    return Buffer.concat([iv, cipher.getAuthTag(), ciphertext]);
+};
+exports.seal = seal;
+const unseal = (payload, key) => {
+    if (payload.length < IV_BYTES + AUTH_TAG_BYTES) {
+        throw new Error('Malformed sealed payload');
+    }
+    const iv = payload.subarray(0, IV_BYTES);
+    const authTag = payload.subarray(IV_BYTES, IV_BYTES + AUTH_TAG_BYTES);
+    const ciphertext = payload.subarray(IV_BYTES + AUTH_TAG_BYTES);
+    const decipher = crypto.createDecipheriv(ALGORITHM, key, iv);
+    decipher.setAuthTag(authTag);
+    return Buffer.concat([decipher.update(ciphertext), decipher.final()]);
+};
+exports.unseal = unseal;
+const generateArchiveKey = () => crypto.randomBytes(exports.ARCHIVE_KEY_BYTES);
+exports.generateArchiveKey = generateArchiveKey;
+const parseMasterKey = (raw, variableName) => {
+    const key = Buffer.from(raw, 'hex');
+    if (key.length !== exports.ARCHIVE_KEY_BYTES) {
+        throw new Error(`${variableName} must be ${exports.ARCHIVE_KEY_BYTES} bytes of hex (${exports.ARCHIVE_KEY_BYTES * 2} characters), got ${key.length} bytes`);
+    }
+    return key;
+};
+exports.parseMasterKey = parseMasterKey;
+const wrapKey = (organizationKey, masterKey) => (0, exports.seal)(organizationKey, masterKey).toString('base64');
+exports.wrapKey = wrapKey;
+const unwrapKey = (wrappedKey, masterKey) => (0, exports.unseal)(Buffer.from(wrappedKey, 'base64'), masterKey);
+exports.unwrapKey = unwrapKey;
+//# sourceMappingURL=archive-seal.util.js.map

package/dist/server/archive/archive-seal.util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"archive-seal.util.js","sourceRoot":"","sources":["../../../src/server/archive/archive-seal.util.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+CAAiC;AAUjC,MAAM,SAAS,GAAG,aAAa,CAAC;AACnB,QAAA,iBAAiB,GAAG,EAAE,CAAC;AACpC,MAAM,QAAQ,GAAG,EAAE,CAAC;AACpB,MAAM,cAAc,GAAG,EAAE,CAAC;AAGnB,MAAM,IAAI,GAAG,CAAC,SAAiB,EAAE,GAAW,EAAU,EAAE;IAC7D,MAAM,EAAE,GAAG,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,CAAC,cAAc,CAAC,SAAS,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;IACzD,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAE7E,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,MAAM,CAAC,UAAU,EAAE,EAAE,UAAU,CAAC,CAAC,CAAC;AAC9D,CAAC,CAAC;AANW,QAAA,IAAI,QAMf;AAEK,MAAM,MAAM,GAAG,CAAC,OAAe,EAAE,GAAW,EAAU,EAAE;IAC7D,IAAI,OAAO,CAAC,MAAM,GAAG,QAAQ,GAAG,cAAc,EAAE,CAAC;QAC/C,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACzC,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,GAAG,cAAc,CAAC,CAAC;IACtE,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC,QAAQ,GAAG,cAAc,CAAC,CAAC;IAE/D,MAAM,QAAQ,GAAG,MAAM,CAAC,gBAAgB,CAAC,SAAS,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;IAC7D,QAAQ,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAK7B,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AACxE,CAAC,CAAC;AAhBW,QAAA,MAAM,UAgBjB;AAGK,MAAM,kBAAkB,GAAG,GAAW,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,yBAAiB,CAAC,CAAC;AAAzE,QAAA,kBAAkB,sBAAuD;AAM/E,MAAM,cAAc,GAAG,CAAC,GAAW,EAAE,YAAoB,EAAU,EAAE;IAC1E,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAEpC,IAAI,GAAG,CAAC,MAAM,KAAK,yBAAiB,EAAE,CAAC;QAGrC,MAAM,IAAI,KAAK,CACb,GAAG,YAAY,YAAY,yBAAiB,kBAC1C,yBAAiB,GAAG,CACtB,qBAAqB,GAAG,CAAC,MAAM,QAAQ,CACxC,CAAC;IACJ,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAdW,QAAA,cAAc,kBAczB;AAGK,MAAM,OAAO,GAAG,CAAC,eAAuB,EAAE,SAAiB,EAAU,EAAE,CAC5E,IAAA,YAAI,EAAC,eAAe,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;AADzC,QAAA,OAAO,WACkC;AAE/C,MAAM,SAAS,GAAG,CAAC,UAAkB,EAAE,SAAiB,EAAU,EAAE,CACzE,IAAA,cAAM,EAAC,MAAM,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,EAAE,SAAS,CAAC,CAAC;AAD1C,QAAA,SAAS,aACiC"}

package/dist/server/archive/archive-stream.service.d.ts

@@ -0,0 +1,31 @@
+import type { ArchiveUploadSession } from './archive-upload-session';
+export type ArchiveRow = Record<string, unknown>;
+export interface ArchivePart {
+    partNumber: number;
+    bytes: number;
+    rowCount: number;
+    firstTimestamp: string;
+}
+export interface ArchiveWriteResult {
+    parts: ArchivePart[];
+    totalRows: number;
+    totalBytes: number;
+    sha256: string;
+}
+export interface ArchiveWriteOptions {
+    key: Buffer;
+    session: Pick<ArchiveUploadSession, 'putPart'>;
+    timeColumn: string;
+    compressionLevel: number;
+    partMaxBytes: number;
+}
+export declare const ARCHIVE_PART_MAX_BYTES: number;
+export declare const ARCHIVE_MAX_PARTS = 10000;
+export declare const ARCHIVE_ROW_BATCH_SIZE = 1000;
+export declare const SEAL_OVERHEAD_BYTES: number;
+export declare const writeArchiveStream: (batches: AsyncIterable<ArchiveRow[]>, options: ArchiveWriteOptions) => Promise<ArchiveWriteResult>;
+export declare function decodeArchiveFrame(sealedFrame: Buffer, key: Buffer): AsyncGenerator<ArchiveRow>;
+export declare function splitArchiveFrames(source: AsyncIterable<Buffer>, parts: ArchivePart[]): AsyncGenerator<{
+    part: ArchivePart;
+    sealed: Buffer;
+}>;

package/dist/server/archive/archive-stream.service.js

@@ -0,0 +1,259 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.writeArchiveStream = exports.SEAL_OVERHEAD_BYTES = exports.ARCHIVE_ROW_BATCH_SIZE = exports.ARCHIVE_MAX_PARTS = exports.ARCHIVE_PART_MAX_BYTES = void 0;
+exports.decodeArchiveFrame = decodeArchiveFrame;
+exports.splitArchiveFrames = splitArchiveFrames;
+const crypto = __importStar(require("crypto"));
+const events_1 = require("events");
+const promises_1 = require("stream/promises");
+const zstd_util_1 = require("./zstd.util");
+const archive_seal_util_1 = require("./archive-seal.util");
+exports.ARCHIVE_PART_MAX_BYTES = 16 * 1024 * 1024;
+exports.ARCHIVE_MAX_PARTS = 10_000;
+exports.ARCHIVE_ROW_BATCH_SIZE = 1000;
+exports.SEAL_OVERHEAD_BYTES = 12 + 16;
+const NDJSON_SEPARATOR = '\n';
+const NDJSON_SEPARATOR_BYTE = 0x0a;
+const COMPRESSOR_WRITE_BYTES = 64 * 1024;
+const ZSTD_BLOCK_SIZE_MAX = 128 * 1024;
+const ZSTD_FRAME_EPILOGUE_BYTES = 3 + 4;
+const zstdCompressBound = (size) => size +
+    Math.floor(size / 256) +
+    (size < ZSTD_BLOCK_SIZE_MAX ? Math.floor((ZSTD_BLOCK_SIZE_MAX - size) / 2048) : 0);
+const timestampOf = (row, timeColumn) => {
+    const value = row[timeColumn];
+    const date = value instanceof Date ? value : typeof value === 'string' ? new Date(value) : undefined;
+    if (!date || Number.isNaN(date.getTime())) {
+        throw new Error(`Archive row has no usable ${timeColumn} (got ${String(value)})`);
+    }
+    return date;
+};
+class ZstdFrame {
+    constructor(compressionLevel, firstTimestamp) {
+        this.firstTimestamp = firstTimestamp;
+        this.rowCount = 0;
+        this.output = [];
+        this.emittedBytes = 0;
+        this.unflushedRawBytes = 0;
+        this.pendingLines = [];
+        this.pendingBytes = 0;
+        this.failure = null;
+        this.compressor = zstd_util_1.zstd.createZstdCompress((0, zstd_util_1.zstdCompressionOptions)(compressionLevel));
+        this.compressor.on('data', (chunk) => {
+            this.output.push(chunk);
+            this.emittedBytes += chunk.length;
+        });
+        this.compressor.on('error', (error) => {
+            this.failure = error;
+        });
+    }
+    async fits(lineBytes, budget) {
+        if (this.upperBound(lineBytes) <= budget) {
+            return true;
+        }
+        await this.flush();
+        return this.upperBound(lineBytes) <= budget;
+    }
+    async append(line, lineBytes) {
+        this.throwIfFailed();
+        this.pendingLines.push(line);
+        this.pendingBytes += lineBytes;
+        this.unflushedRawBytes += lineBytes;
+        this.rowCount += 1;
+        if (this.pendingBytes >= COMPRESSOR_WRITE_BYTES) {
+            await this.writePending();
+        }
+    }
+    async finish() {
+        await this.writePending();
+        this.compressor.end();
+        await (0, promises_1.finished)(this.compressor);
+        this.throwIfFailed();
+        return Buffer.concat(this.output);
+    }
+    destroy() {
+        this.compressor.destroy();
+    }
+    upperBound(extraRawBytes) {
+        return (this.emittedBytes +
+            zstdCompressBound(this.unflushedRawBytes + extraRawBytes) +
+            ZSTD_FRAME_EPILOGUE_BYTES);
+    }
+    async writePending() {
+        this.throwIfFailed();
+        if (this.pendingBytes === 0) {
+            return;
+        }
+        const chunk = Buffer.from(this.pendingLines.join(''), 'utf8');
+        this.pendingLines = [];
+        this.pendingBytes = 0;
+        if (!this.compressor.write(chunk)) {
+            await (0, events_1.once)(this.compressor, 'drain');
+        }
+    }
+    async flush() {
+        await this.writePending();
+        await new Promise((resolve, reject) => {
+            this.compressor.flush((error) => (error ? reject(error) : resolve()));
+        });
+        while (this.compressor.readableLength > 0) {
+            await (0, events_1.once)(this.compressor, 'data');
+        }
+        this.throwIfFailed();
+        this.unflushedRawBytes = 0;
+    }
+    throwIfFailed() {
+        if (this.failure) {
+            throw this.failure;
+        }
+    }
+}
+const writeArchiveStream = async (batches, options) => {
+    const { key, session, timeColumn, compressionLevel, partMaxBytes } = options;
+    const frameBudget = partMaxBytes - exports.SEAL_OVERHEAD_BYTES;
+    if (frameBudget <= 0) {
+        throw new Error(`Archive part size ${partMaxBytes} leaves no room for a frame after the ${exports.SEAL_OVERHEAD_BYTES}-byte seal`);
+    }
+    const digest = crypto.createHash('sha256');
+    const parts = [];
+    let frame = null;
+    let previousDateMs = Number.NEGATIVE_INFINITY;
+    const closeFrame = async (current) => {
+        const sealed = (0, archive_seal_util_1.seal)(await current.finish(), key);
+        const partNumber = parts.length + 1;
+        if (sealed.length > partMaxBytes) {
+            throw new Error(`Archive part ${partNumber} seals to ${sealed.length} bytes, above the ${partMaxBytes}-byte part size: a single row larger than a part cannot be archived`);
+        }
+        if (partNumber > exports.ARCHIVE_MAX_PARTS) {
+            throw new Error(`Archive object needs more than ${exports.ARCHIVE_MAX_PARTS} parts, the most one AbyssStorage upload session accepts`);
+        }
+        await session.putPart(partNumber, sealed);
+        digest.update(sealed);
+        parts.push({
+            partNumber,
+            bytes: sealed.length,
+            rowCount: current.rowCount,
+            firstTimestamp: current.firstTimestamp,
+        });
+    };
+    try {
+        for await (const batch of batches) {
+            for (const row of batch) {
+                const date = timestampOf(row, timeColumn);
+                if (date.getTime() < previousDateMs) {
+                    throw new Error(`Archive rows must arrive ordered by ${timeColumn}: ${date.toISOString()} came after ${new Date(previousDateMs).toISOString()}`);
+                }
+                previousDateMs = date.getTime();
+                const line = `${JSON.stringify(row)}${NDJSON_SEPARATOR}`;
+                const lineBytes = Buffer.byteLength(line, 'utf8');
+                if (frame && !(await frame.fits(lineBytes, frameBudget))) {
+                    await closeFrame(frame);
+                    frame = null;
+                }
+                if (!frame) {
+                    frame = new ZstdFrame(compressionLevel, date.toISOString());
+                }
+                await frame.append(line, lineBytes);
+            }
+        }
+        if (frame) {
+            await closeFrame(frame);
+            frame = null;
+        }
+    }
+    catch (error) {
+        frame?.destroy();
+        throw error;
+    }
+    return {
+        parts,
+        totalRows: parts.reduce((total, part) => total + part.rowCount, 0),
+        totalBytes: parts.reduce((total, part) => total + part.bytes, 0),
+        sha256: digest.digest('hex'),
+    };
+};
+exports.writeArchiveStream = writeArchiveStream;
+async function* decodeArchiveFrame(sealedFrame, key) {
+    const decompressor = zstd_util_1.zstd.createZstdDecompress();
+    decompressor.end((0, archive_seal_util_1.unseal)(sealedFrame, key));
+    let carry = [];
+    for await (const chunk of decompressor) {
+        let start = 0;
+        let separator = chunk.indexOf(NDJSON_SEPARATOR_BYTE, start);
+        while (separator !== -1) {
+            const tail = chunk.subarray(start, separator);
+            const line = carry.length === 0 ? tail : Buffer.concat([...carry, tail]);
+            carry = [];
+            yield JSON.parse(line.toString('utf8'));
+            start = separator + 1;
+            separator = chunk.indexOf(NDJSON_SEPARATOR_BYTE, start);
+        }
+        if (start < chunk.length) {
+            carry.push(chunk.subarray(start));
+        }
+    }
+    if (carry.length > 0) {
+        throw new Error('Archive frame ends inside a row');
+    }
+}
+async function* splitArchiveFrames(source, parts) {
+    let index = 0;
+    let held = [];
+    let heldBytes = 0;
+    for await (const chunk of source) {
+        let offset = 0;
+        while (offset < chunk.length) {
+            const part = parts[index];
+            if (!part) {
+                throw new Error(`Archive object is longer than its ${parts.length}-part index`);
+            }
+            const take = Math.min(part.bytes - heldBytes, chunk.length - offset);
+            held.push(chunk.subarray(offset, offset + take));
+            heldBytes += take;
+            offset += take;
+            if (heldBytes === part.bytes) {
+                yield { part, sealed: Buffer.concat(held) };
+                held = [];
+                heldBytes = 0;
+                index += 1;
+            }
+        }
+    }
+    if (index !== parts.length) {
+        throw new Error(`Archive object ends inside part ${parts[index].partNumber}: ${heldBytes} of its ${parts[index].bytes} indexed bytes arrived`);
+    }
+}
+//# sourceMappingURL=archive-stream.service.js.map

package/dist/server/archive/archive-stream.service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"archive-stream.service.js","sourceRoot":"","sources":["../../../src/server/archive/archive-stream.service.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA+VA,gDAmCC;AAaD,gDAoCC;AAnbD,+CAAiC;AACjC,mCAA8B;AAC9B,8CAA2C;AAC3C,2CAA+E;AAC/E,2DAAmD;AA6DtC,QAAA,sBAAsB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAG1C,QAAA,iBAAiB,GAAG,MAAM,CAAC;AAG3B,QAAA,sBAAsB,GAAG,IAAI,CAAC;AAG9B,QAAA,mBAAmB,GAAG,EAAE,GAAG,EAAE,CAAC;AAI3C,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAC9B,MAAM,qBAAqB,GAAG,IAAI,CAAC;AAKnC,MAAM,sBAAsB,GAAG,EAAE,GAAG,IAAI,CAAC;AAGzC,MAAM,mBAAmB,GAAG,GAAG,GAAG,IAAI,CAAC;AAQvC,MAAM,yBAAyB,GAAG,CAAC,GAAG,CAAC,CAAC;AAGxC,MAAM,iBAAiB,GAAG,CAAC,IAAY,EAAU,EAAE,CACjD,IAAI;IACJ,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,GAAG,CAAC;IACtB,CAAC,IAAI,GAAG,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,mBAAmB,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAIrF,MAAM,WAAW,GAAG,CAAC,GAAe,EAAE,UAAkB,EAAQ,EAAE;IAChE,MAAM,KAAK,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC;IAC9B,MAAM,IAAI,GACR,KAAK,YAAY,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAE1F,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;QAG1C,MAAM,IAAI,KAAK,CAAC,6BAA6B,UAAU,SAAS,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACpF,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAaF,MAAM,SAAS;IAab,YAAY,gBAAwB,EAAkB,cAAsB;QAAtB,mBAAc,GAAd,cAAc,CAAQ;QAZrE,aAAQ,GAAG,CAAC,CAAC;QAGH,WAAM,GAAa,EAAE,CAAC;QAE/B,iBAAY,GAAG,CAAC,CAAC;QAEjB,sBAAiB,GAAG,CAAC,CAAC;QACtB,iBAAY,GAAa,EAAE,CAAC;QAC5B,iBAAY,GAAG,CAAC,CAAC;QACjB,YAAO,GAAiB,IAAI,CAAC;QAGnC,IAAI,CAAC,UAAU,GAAG,gBAAI,CAAC,kBAAkB,CAAC,IAAA,kCAAsB,EAAC,gBAAgB,CAAC,CAAC,CAAC;QACpF,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;YAC3C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACxB,IAAI,CAAC,YAAY,IAAI,KAAK,CAAC,MAAM,CAAC;QACpC,CAAC,CAAC,CAAC;QAGH,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE;YAC3C,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACvB,CAAC,CAAC,CAAC;IACL,CAAC;IAGM,KAAK,CAAC,IAAI,CAAC,SAAiB,EAAE,MAAc;QACjD,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,MAAM,EAAE,CAAC;YACzC,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,OAAO,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,MAAM,CAAC;IAC9C,CAAC;IAEM,KAAK,CAAC,MAAM,CAAC,IAAY,EAAE,SAAiB;QACjD,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,CAAC,YAAY,IAAI,SAAS,CAAC;QAC/B,IAAI,CAAC,iBAAiB,IAAI,SAAS,CAAC;QACpC,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;QAEnB,IAAI,IAAI,CAAC,YAAY,IAAI,sBAAsB,EAAE,CAAC;YAChD,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC5B,CAAC;IACH,CAAC;IAGM,KAAK,CAAC,MAAM;QACjB,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1B,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;QACtB,MAAM,IAAA,mBAAQ,EAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAChC,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACpC,CAAC;IAEM,OAAO;QACZ,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;IAC5B,CAAC;IAEO,UAAU,CAAC,aAAqB;QACtC,OAAO,CACL,IAAI,CAAC,YAAY;YACjB,iBAAiB,CAAC,IAAI,CAAC,iBAAiB,GAAG,aAAa,CAAC;YACzD,yBAAyB,CAC1B,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,IAAI,IAAI,CAAC,YAAY,KAAK,CAAC,EAAE,CAAC;YAC5B,OAAO;QACT,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QAC9D,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;QAItB,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YAClC,MAAM,IAAA,aAAI,EAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,KAAK;QACjB,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC1B,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC1C,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACxE,CAAC,CAAC,CAAC;QAIH,OAAO,IAAI,CAAC,UAAU,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;YAC1C,MAAM,IAAA,aAAI,EAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,CAAC,aAAa,EAAE,CAAC;QACrB,IAAI,CAAC,iBAAiB,GAAG,CAAC,CAAC;IAC7B,CAAC;IAEO,aAAa;QACnB,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC;QACrB,CAAC;IACH,CAAC;CACF;AAYM,MAAM,kBAAkB,GAAG,KAAK,EACrC,OAAoC,EACpC,OAA4B,EACC,EAAE;IAC/B,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC;IAC7E,MAAM,WAAW,GAAG,YAAY,GAAG,2BAAmB,CAAC;IACvD,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CACb,qBAAqB,YAAY,yCAAyC,2BAAmB,YAAY,CAC1G,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAkB,EAAE,CAAC;IAChC,IAAI,KAAK,GAAqB,IAAI,CAAC;IACnC,IAAI,cAAc,GAAG,MAAM,CAAC,iBAAiB,CAAC;IAE9C,MAAM,UAAU,GAAG,KAAK,EAAE,OAAkB,EAAiB,EAAE;QAC7D,MAAM,MAAM,GAAG,IAAA,wBAAI,EAAC,MAAM,OAAO,CAAC,MAAM,EAAE,EAAE,GAAG,CAAC,CAAC;QACjD,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;QAEpC,IAAI,MAAM,CAAC,MAAM,GAAG,YAAY,EAAE,CAAC;YAIjC,MAAM,IAAI,KAAK,CACb,gBAAgB,UAAU,aAAa,MAAM,CAAC,MAAM,qBAAqB,YAAY,qEAAqE,CAC3J,CAAC;QACJ,CAAC;QACD,IAAI,UAAU,GAAG,yBAAiB,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CACb,kCAAkC,yBAAiB,0DAA0D,CAC9G,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QAC1C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACtB,KAAK,CAAC,IAAI,CAAC;YACT,UAAU;YACV,KAAK,EAAE,MAAM,CAAC,MAAM;YACpB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,cAAc,EAAE,OAAO,CAAC,cAAc;SACvC,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,IAAI,CAAC;QACH,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAClC,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACxB,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;gBAC1C,IAAI,IAAI,CAAC,OAAO,EAAE,GAAG,cAAc,EAAE,CAAC;oBACpC,MAAM,IAAI,KAAK,CACb,uCAAuC,UAAU,KAAK,IAAI,CAAC,WAAW,EAAE,eAAe,IAAI,IAAI,CAC7F,cAAc,CACf,CAAC,WAAW,EAAE,EAAE,CAClB,CAAC;gBACJ,CAAC;gBACD,cAAc,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;gBAEhC,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,gBAAgB,EAAE,CAAC;gBACzD,MAAM,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;gBAElD,IAAI,KAAK,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC;oBACzD,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;oBACxB,KAAK,GAAG,IAAI,CAAC;gBACf,CAAC;gBAED,IAAI,CAAC,KAAK,EAAE,CAAC;oBACX,KAAK,GAAG,IAAI,SAAS,CAAC,gBAAgB,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;gBAC9D,CAAC;gBACD,MAAM,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QAED,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,UAAU,CAAC,KAAK,CAAC,CAAC;YACxB,KAAK,GAAG,IAAI,CAAC;QACf,CAAC;IACH,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,KAAK,EAAE,OAAO,EAAE,CAAC;QACjB,MAAM,KAAK,CAAC;IACd,CAAC;IAED,OAAO;QACL,KAAK;QACL,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;QAClE,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;QAChE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC;KAC7B,CAAC;AACJ,CAAC,CAAC;AAxFW,QAAA,kBAAkB,sBAwF7B;AAgBK,KAAK,SAAS,CAAC,CAAC,kBAAkB,CACvC,WAAmB,EACnB,GAAW;IAEX,MAAM,YAAY,GAAG,gBAAI,CAAC,oBAAoB,EAAE,CAAC;IACjD,YAAY,CAAC,GAAG,CAAC,IAAA,0BAAM,EAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC;IAK3C,IAAI,KAAK,GAAa,EAAE,CAAC;IAEzB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,YAAqC,EAAE,CAAC;QAChE,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,qBAAqB,EAAE,KAAK,CAAC,CAAC;QAE5D,OAAO,SAAS,KAAK,CAAC,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;YAC9C,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC;YACzE,KAAK,GAAG,EAAE,CAAC;YACX,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAe,CAAC;YACtD,KAAK,GAAG,SAAS,GAAG,CAAC,CAAC;YACtB,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,qBAAqB,EAAE,KAAK,CAAC,CAAC;QAC1D,CAAC;QAED,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAGrB,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,CAAC;AACH,CAAC;AAaM,KAAK,SAAS,CAAC,CAAC,kBAAkB,CACvC,MAA6B,EAC7B,KAAoB;IAEpB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,IAAI,GAAa,EAAE,CAAC;IACxB,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACjC,IAAI,MAAM,GAAG,CAAC,CAAC;QAEf,OAAO,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC7B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;YAC1B,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,MAAM,IAAI,KAAK,CAAC,qCAAqC,KAAK,CAAC,MAAM,aAAa,CAAC,CAAC;YAClF,CAAC;YAED,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,SAAS,EAAE,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;YACrE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC;YACjD,SAAS,IAAI,IAAI,CAAC;YAClB,MAAM,IAAI,IAAI,CAAC;YAEf,IAAI,SAAS,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC7B,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5C,IAAI,GAAG,EAAE,CAAC;gBACV,SAAS,GAAG,CAAC,CAAC;gBACd,KAAK,IAAI,CAAC,CAAC;YACb,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,KAAK,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CACb,mCAAmC,KAAK,CAAC,KAAK,CAAC,CAAC,UAAU,KAAK,SAAS,WAAW,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,wBAAwB,CAC9H,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/server/archive/archive-upload-session.d.ts

@@ -0,0 +1,5 @@
+export interface ArchiveUploadSession {
+    putPart(partNumber: number, payload: Buffer): Promise<void>;
+    complete(): Promise<string>;
+    abort(): Promise<void>;
+}

package/dist/server/archive/archive-upload-session.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=archive-upload-session.js.map

package/dist/server/archive/archive-upload-session.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"archive-upload-session.js","sourceRoot":"","sources":["../../../src/server/archive/archive-upload-session.ts"],"names":[],"mappings":""}

package/dist/server/archive/index.d.ts

@@ -0,0 +1,4 @@
+export * from './archive-seal.util';
+export * from './archive-stream.service';
+export * from './archive-upload-session';
+export * from './zstd.util';

package/dist/server/archive/index.js

@@ -0,0 +1,21 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./archive-seal.util"), exports);
+__exportStar(require("./archive-stream.service"), exports);
+__exportStar(require("./archive-upload-session"), exports);
+__exportStar(require("./zstd.util"), exports);
+//# sourceMappingURL=index.js.map

package/dist/server/archive/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/server/archive/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,sDAAoC;AACpC,2DAAyC;AACzC,2DAAyC;AACzC,8CAA4B"}

package/dist/server/archive/tests/archive-fixture.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/server/archive/tests/archive-fixture.spec.js

@@ -0,0 +1,84 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const crypto = __importStar(require("crypto"));
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const stream_1 = require("stream");
+const archive_seal_util_1 = require("../archive-seal.util");
+const archive_stream_service_1 = require("../archive-stream.service");
+const FIXTURES = path.join(__dirname, 'fixtures');
+const object = fs.readFileSync(path.join(FIXTURES, 'archive-object.bin'));
+const expected = JSON.parse(fs.readFileSync(path.join(FIXTURES, 'archive-object.expected.json'), 'utf8'));
+const key = Buffer.from(expected.keyHex, 'hex');
+const decodeAll = async (source) => {
+    const rows = [];
+    for await (const { part, sealed } of (0, archive_stream_service_1.splitArchiveFrames)(source, expected.parts)) {
+        const decoded = [];
+        for await (const row of (0, archive_stream_service_1.decodeArchiveFrame)(sealed, key)) {
+            decoded.push(row);
+        }
+        expect(decoded).toHaveLength(part.rowCount);
+        expect(decoded[0].dateValue).toBe(part.firstTimestamp);
+        rows.push(...decoded);
+    }
+    return rows;
+};
+describe('archive object written by Monitor before the move', () => {
+    it('has the shape the fixture claims', () => {
+        expect(expected.parts.length).toBeGreaterThanOrEqual(2);
+        expect(object.length).toBe(expected.totalBytes);
+        expect(expected.parts.reduce((total, part) => total + part.bytes, 0)).toBe(object.length);
+    });
+    it('has the sha256 the writer recorded', () => {
+        expect(crypto.createHash('sha256').update(object).digest('hex')).toBe(expected.sha256);
+    });
+    it('unwraps the stored organization key with the master key', () => {
+        expect((0, archive_seal_util_1.unwrapKey)(expected.wrappedKey, Buffer.from(expected.masterKey, 'hex')).toString('hex')).toBe(expected.keyHex);
+    });
+    it('reads back to the exact rows, part by part', async () => {
+        const rows = await decodeAll(stream_1.Readable.from([object]));
+        expect(rows).toEqual(expected.rows);
+        expect(rows).toHaveLength(expected.totalRows);
+    });
+    it('is read the same when the stream arrives in arbitrary chunks', async () => {
+        const chunks = [];
+        for (let offset = 0; offset < object.length; offset += 97) {
+            chunks.push(object.subarray(offset, offset + 97));
+        }
+        expect(await decodeAll(stream_1.Readable.from(chunks))).toEqual(expected.rows);
+    });
+});
+//# sourceMappingURL=archive-fixture.spec.js.map

package/dist/server/archive/tests/archive-fixture.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"archive-fixture.spec.js","sourceRoot":"","sources":["../../../../src/server/archive/tests/archive-fixture.spec.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+CAAiC;AACjC,uCAAyB;AACzB,2CAA6B;AAC7B,mCAAkC;AAClC,4DAAiD;AACjD,sEAKmC;AAwBnC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;AAClD,MAAM,MAAM,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,oBAAoB,CAAC,CAAC,CAAC;AAC1E,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CACzB,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,8BAA8B,CAAC,EAAE,MAAM,CAAC,CACjE,CAAC;AACd,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AAEhD,MAAM,SAAS,GAAG,KAAK,EAAE,MAAgB,EAAyB,EAAE;IAClE,MAAM,IAAI,GAAiB,EAAE,CAAC;IAC9B,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,IAAA,2CAAkB,EAAC,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAChF,MAAM,OAAO,GAAiB,EAAE,CAAC;QACjC,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAA,2CAAkB,EAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;YACxD,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;QACD,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5C,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACvD,IAAI,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;IACxB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,QAAQ,CAAC,mDAAmD,EAAE,GAAG,EAAE;IACjE,EAAE,CAAC,kCAAkC,EAAE,GAAG,EAAE;QAC1C,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QACxD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAChD,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC5F,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACzF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACjE,MAAM,CACJ,IAAA,6BAAS,EAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CACvF,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC1B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4CAA4C,EAAE,KAAK,IAAI,EAAE;QAC1D,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,iBAAQ,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAEtD,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACpC,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;QAC5E,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,IAAI,EAAE,EAAE,CAAC;YAC1D,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC;QACpD,CAAC;QAED,MAAM,CAAC,MAAM,SAAS,CAAC,iBAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IACxE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/server/archive/tests/archive-seal.util.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/server/archive/tests/archive-seal.util.spec.js

@@ -0,0 +1,98 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const crypto = __importStar(require("crypto"));
+const archive_seal_util_1 = require("../archive-seal.util");
+describe('seal / unseal', () => {
+    it('round-trips', () => {
+        const key = crypto.randomBytes(32);
+        const plaintext = Buffer.from('a log line with an Authorization header in it');
+        expect((0, archive_seal_util_1.unseal)((0, archive_seal_util_1.seal)(plaintext, key), key)).toEqual(plaintext);
+    });
+    it('produces an exact frame length: iv (12 o) + authTag (16 o) + ciphertext', () => {
+        const key = crypto.randomBytes(32);
+        const plaintext = crypto.randomBytes(1024 * 1024);
+        const sealed = (0, archive_seal_util_1.seal)(plaintext, key);
+        expect(sealed.length).toBe(12 + 16 + plaintext.length);
+        expect((0, archive_seal_util_1.unseal)(sealed, key)).toEqual(plaintext);
+    });
+    it('produces a different payload every time, so identical inputs do not look identical', () => {
+        const key = crypto.randomBytes(32);
+        const plaintext = Buffer.from('same input');
+        expect((0, archive_seal_util_1.seal)(plaintext, key)).not.toEqual((0, archive_seal_util_1.seal)(plaintext, key));
+    });
+    it('refuses a tampered payload instead of returning garbage', () => {
+        const key = crypto.randomBytes(32);
+        const sealed = (0, archive_seal_util_1.seal)(Buffer.from('trustworthy'), key);
+        const tampered = Buffer.from(sealed);
+        tampered[tampered.length - 1] ^= 0xff;
+        expect(() => (0, archive_seal_util_1.unseal)(tampered, key)).toThrow();
+    });
+    it('refuses the wrong key', () => {
+        const sealed = (0, archive_seal_util_1.seal)(Buffer.from('secret'), crypto.randomBytes(32));
+        expect(() => (0, archive_seal_util_1.unseal)(sealed, crypto.randomBytes(32))).toThrow();
+    });
+    it('refuses a payload shorter than its iv and tag', () => {
+        expect(() => (0, archive_seal_util_1.unseal)(Buffer.alloc(27), crypto.randomBytes(32))).toThrow('Malformed sealed payload');
+    });
+});
+describe('organization key wrapping', () => {
+    it('generates 32-byte keys, different every time', () => {
+        const a = (0, archive_seal_util_1.generateArchiveKey)();
+        expect(a.length).toBe(32);
+        expect((0, archive_seal_util_1.generateArchiveKey)()).not.toEqual(a);
+    });
+    it('wraps to base64 and unwraps to the same key, without storing it in the clear', () => {
+        const master = crypto.randomBytes(32);
+        const key = (0, archive_seal_util_1.generateArchiveKey)();
+        const wrapped = (0, archive_seal_util_1.wrapKey)(key, master);
+        const raw = Buffer.from(wrapped, 'base64');
+        expect((0, archive_seal_util_1.unwrapKey)(wrapped, master)).toEqual(key);
+        expect(raw.includes(key)).toBe(false);
+        expect(raw.length).toBe(12 + 16 + key.length);
+    });
+    it('does not unwrap with another master key', () => {
+        const wrapped = (0, archive_seal_util_1.wrapKey)((0, archive_seal_util_1.generateArchiveKey)(), crypto.randomBytes(32));
+        expect(() => (0, archive_seal_util_1.unwrapKey)(wrapped, crypto.randomBytes(32))).toThrow();
+    });
+    it('parses a 32-byte hex master key', () => {
+        const master = crypto.randomBytes(32);
+        expect((0, archive_seal_util_1.parseMasterKey)(master.toString('hex'), 'SOME_MASTER_KEY')).toEqual(master);
+    });
+    it('refuses a short master key and names the variable', () => {
+        expect(() => (0, archive_seal_util_1.parseMasterKey)('abcd', 'SOME_MASTER_KEY')).toThrow('SOME_MASTER_KEY must be 32 bytes of hex (64 characters), got 2 bytes');
+    });
+});
+//# sourceMappingURL=archive-seal.util.spec.js.map

package/dist/server/archive/tests/archive-seal.util.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"archive-seal.util.spec.js","sourceRoot":"","sources":["../../../../src/server/archive/tests/archive-seal.util.spec.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+CAAiC;AACjC,4DAO8B;AAE9B,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,EAAE,CAAC,aAAa,EAAE,GAAG,EAAE;QACrB,MAAM,GAAG,GAAG,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QACnC,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;QAE/E,MAAM,CAAC,IAAA,0BAAM,EAAC,IAAA,wBAAI,EAAC,SAAS,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yEAAyE,EAAE,GAAG,EAAE;QACjF,MAAM,GAAG,GAAG,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QACnC,MAAM,SAAS,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;QAElD,MAAM,MAAM,GAAG,IAAA,wBAAI,EAAC,SAAS,EAAE,GAAG,CAAC,CAAC;QAEpC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;QACvD,MAAM,CAAC,IAAA,0BAAM,EAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oFAAoF,EAAE,GAAG,EAAE;QAC5F,MAAM,GAAG,GAAG,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QACnC,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAI5C,MAAM,CAAC,IAAA,wBAAI,EAAC,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,IAAA,wBAAI,EAAC,SAAS,EAAE,GAAG,CAAC,CAAC,CAAC;IACjE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACjE,MAAM,GAAG,GAAG,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QACnC,MAAM,MAAM,GAAG,IAAA,wBAAI,EAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,GAAG,CAAC,CAAC;QACrD,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAGrC,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC;QAItC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAA,0BAAM,EAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uBAAuB,EAAE,GAAG,EAAE;QAC/B,MAAM,MAAM,GAAG,IAAA,wBAAI,EAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC;QAEnE,MAAM,CAAC,GAAG,EAAE,CAAC,IAAA,0BAAM,EAAC,MAAM,EAAE,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;IACjE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACvD,MAAM,CAAC,GAAG,EAAE,CAAC,IAAA,0BAAM,EAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CACpE,0BAA0B,CAC3B,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;IACzC,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,CAAC,GAAG,IAAA,sCAAkB,GAAE,CAAC;QAE/B,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1B,MAAM,CAAC,IAAA,sCAAkB,GAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8EAA8E,EAAE,GAAG,EAAE;QACtF,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QACtC,MAAM,GAAG,GAAG,IAAA,sCAAkB,GAAE,CAAC;QAEjC,MAAM,OAAO,GAAG,IAAA,2BAAO,EAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACrC,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAE3C,MAAM,CAAC,IAAA,6BAAS,EAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAChD,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAEtC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;QACjD,MAAM,OAAO,GAAG,IAAA,2BAAO,EAAC,IAAA,sCAAkB,GAAE,EAAE,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC;QAEtE,MAAM,CAAC,GAAG,EAAE,CAAC,IAAA,6BAAS,EAAC,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;IACrE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;QACzC,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QAEtC,MAAM,CAAC,IAAA,kCAAc,EAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,iBAAiB,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACpF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,CAAC,GAAG,EAAE,CAAC,IAAA,kCAAc,EAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC,CAAC,OAAO,CAC7D,sEAAsE,CACvE,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/server/archive/tests/archive-stream.service.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/server/archive/tests/archive-stream.service.spec.js

@@ -0,0 +1,309 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const crypto = __importStar(require("crypto"));
+const stream_1 = require("stream");
+const archive_seal_util_1 = require("../archive-seal.util");
+const archive_stream_service_1 = require("../archive-stream.service");
+const COMPRESSION_LEVEL = 1;
+const TIME_COLUMN = 'dateValue';
+const DAY_START = Date.UTC(2026, 0, 10);
+const newKey = () => crypto.randomBytes(32);
+const recordingSession = () => {
+    const payloads = new Map();
+    return {
+        payloads,
+        putPart: jest.fn(async (partNumber, payload) => {
+            payloads.set(partNumber, Buffer.from(payload));
+        }),
+    };
+};
+const noise = (seed, length) => {
+    let text = '';
+    for (let counter = 0; text.length < length; counter += 1) {
+        text += crypto.createHash('sha256').update(`${seed}:${counter}`).digest('base64');
+    }
+    return text.slice(0, length);
+};
+const makeRows = (count, messageLength = () => 24) => Array.from({ length: count }, (_, index) => ({
+    id: `row-${index}`,
+    applicationId: 'application-under-test',
+    level: index % 6,
+    message: `message ${index} ${noise(index, messageLength(index))}`,
+    dateValue: new Date(DAY_START + index * 1000),
+}));
+async function* inBatches(rows, size) {
+    for (let start = 0; start < rows.length; start += size) {
+        yield rows.slice(start, start + size);
+    }
+}
+const asArchived = (rows) => JSON.parse(JSON.stringify(rows));
+const decodeAll = async (sealed, key) => {
+    const rows = [];
+    for await (const row of (0, archive_stream_service_1.decodeArchiveFrame)(sealed, key)) {
+        rows.push(row);
+    }
+    return rows;
+};
+const sha256Of = (buffers) => crypto.createHash('sha256').update(Buffer.concat(buffers)).digest('hex');
+const payloadsInOrder = (session, parts) => parts.map((part) => session.payloads.get(part.partNumber));
+describe('archive-stream.service', () => {
+    describe('writeArchiveStream', () => {
+        it('cuts a day bigger than a part into parts that each decode alone and give the day back in order', async () => {
+            const key = newKey();
+            const session = recordingSession();
+            const rows = makeRows(50_000);
+            const partMaxBytes = 16 * 1024;
+            const result = await (0, archive_stream_service_1.writeArchiveStream)(inBatches(rows, 1000), {
+                key,
+                session,
+                timeColumn: TIME_COLUMN,
+                compressionLevel: COMPRESSION_LEVEL,
+                partMaxBytes,
+            });
+            expect(result.parts.length).toBeGreaterThanOrEqual(3);
+            expect(result.parts.map((part) => part.partNumber)).toEqual(result.parts.map((_, index) => index + 1));
+            expect([...session.payloads.keys()]).toEqual(result.parts.map((part) => part.partNumber));
+            const decoded = [];
+            for (const part of result.parts) {
+                const sealed = session.payloads.get(part.partNumber);
+                const partRows = await decodeAll(sealed, key);
+                expect(sealed.length).toBeLessThanOrEqual(partMaxBytes);
+                expect(part.bytes).toBe(sealed.length);
+                expect(part.rowCount).toBe(partRows.length);
+                expect(part.firstTimestamp).toBe(partRows[0].dateValue);
+                decoded.push(...partRows);
+            }
+            expect(decoded).toEqual(asArchived(rows));
+            expect(result.totalRows).toBe(rows.length);
+            const stored = payloadsInOrder(session, result.parts);
+            expect(result.totalBytes).toBe(stored.reduce((total, sealed) => total + sealed.length, 0));
+            expect(result.sha256).toBe(sha256Of(stored));
+        });
+        it('never holds more rows at once than one batch', async () => {
+            const batchSize = 250;
+            let handedOut = 0;
+            let encoded = 0;
+            let mostAlive = 0;
+            const observe = () => {
+                mostAlive = Math.max(mostAlive, handedOut - encoded);
+            };
+            const rows = makeRows(20_000).map((row) => ({
+                ...row,
+                toJSON: () => {
+                    encoded += 1;
+                    observe();
+                    return row;
+                },
+            }));
+            async function* spiedBatches() {
+                for (let start = 0; start < rows.length; start += batchSize) {
+                    const batch = rows.slice(start, start + batchSize);
+                    handedOut += batch.length;
+                    observe();
+                    yield batch;
+                }
+            }
+            const result = await (0, archive_stream_service_1.writeArchiveStream)(spiedBatches(), {
+                key: newKey(),
+                session: recordingSession(),
+                timeColumn: TIME_COLUMN,
+                compressionLevel: COMPRESSION_LEVEL,
+                partMaxBytes: 16 * 1024,
+            });
+            expect(result.totalRows).toBe(rows.length);
+            expect(encoded).toBe(rows.length);
+            expect(mostAlive).toBe(batchSize);
+        });
+        it('cuts on the bytes written, never on a number of rows', async () => {
+            const session = recordingSession();
+            const partMaxBytes = 16 * 1024;
+            const rows = makeRows(3_000, (index) => 50 + ((index * 7919) % 1500));
+            const result = await (0, archive_stream_service_1.writeArchiveStream)(inBatches(rows, 500), {
+                key: newKey(),
+                session,
+                timeColumn: TIME_COLUMN,
+                compressionLevel: COMPRESSION_LEVEL,
+                partMaxBytes,
+            });
+            const sizes = payloadsInOrder(session, result.parts).map((sealed) => sealed.length);
+            for (const size of sizes) {
+                expect(size).toBeLessThanOrEqual(partMaxBytes);
+            }
+            for (const size of sizes.slice(0, -1)) {
+                expect(size).toBeGreaterThan(partMaxBytes * 0.75);
+            }
+            expect(new Set(result.parts.map((part) => part.rowCount)).size).toBeGreaterThan(1);
+        });
+        it('refuses a single row larger than a part rather than sending an oversized part', async () => {
+            const session = recordingSession();
+            await expect((0, archive_stream_service_1.writeArchiveStream)(inBatches(makeRows(1, () => 8_000), 1), {
+                key: newKey(),
+                session,
+                timeColumn: TIME_COLUMN,
+                compressionLevel: COMPRESSION_LEVEL,
+                partMaxBytes: 2048,
+            })).rejects.toThrow(/above the 2048-byte part size/);
+            expect(session.putPart).not.toHaveBeenCalled();
+        });
+        it('refuses rows that go back in time, rather than writing an index that lies', async () => {
+            const [first, second, ...rest] = makeRows(10);
+            await expect((0, archive_stream_service_1.writeArchiveStream)(inBatches([second, first, ...rest], 10), {
+                key: newKey(),
+                session: recordingSession(),
+                timeColumn: TIME_COLUMN,
+                compressionLevel: COMPRESSION_LEVEL,
+                partMaxBytes: 16 * 1024,
+            })).rejects.toThrow(/ordered by dateValue/);
+        });
+        it("orders and indexes on the table's own time column, which is not always dateValue", async () => {
+            const key = newKey();
+            const session = recordingSession();
+            const rows = Array.from({ length: 3_000 }, (_, index) => ({
+                id: `history-${index}`,
+                applicationCronTaskId: 'cron-task-under-test',
+                startDate: new Date(DAY_START + index * 1000),
+                responseBody: noise(index, 40),
+            }));
+            const result = await (0, archive_stream_service_1.writeArchiveStream)(inBatches(rows, 500), {
+                key,
+                session,
+                timeColumn: 'startDate',
+                compressionLevel: COMPRESSION_LEVEL,
+                partMaxBytes: 8 * 1024,
+            });
+            expect(result.parts.length).toBeGreaterThanOrEqual(3);
+            for (const part of result.parts) {
+                const partRows = await decodeAll(session.payloads.get(part.partNumber), key);
+                expect(part.firstTimestamp).toBe(partRows[0].startDate);
+            }
+            const [first, second, ...rest] = rows;
+            await expect((0, archive_stream_service_1.writeArchiveStream)(inBatches([second, first, ...rest], 500), {
+                key,
+                session: recordingSession(),
+                timeColumn: 'startDate',
+                compressionLevel: COMPRESSION_LEVEL,
+                partMaxBytes: 8 * 1024,
+            })).rejects.toThrow(/ordered by startDate/);
+        });
+        it('refuses a row without a usable dateValue, which the index could not place', async () => {
+            await expect((0, archive_stream_service_1.writeArchiveStream)(inBatches([{ id: 'no-date', message: 'orphan' }], 1), {
+                key: newKey(),
+                session: recordingSession(),
+                timeColumn: TIME_COLUMN,
+                compressionLevel: COMPRESSION_LEVEL,
+                partMaxBytes: 16 * 1024,
+            })).rejects.toThrow(/no usable dateValue/);
+        });
+        it('writes no part for a day without rows', async () => {
+            const session = recordingSession();
+            const result = await (0, archive_stream_service_1.writeArchiveStream)(inBatches([], 10), {
+                key: newKey(),
+                session,
+                timeColumn: TIME_COLUMN,
+                compressionLevel: COMPRESSION_LEVEL,
+                partMaxBytes: 16 * 1024,
+            });
+            expect(result).toEqual({ parts: [], totalRows: 0, totalBytes: 0, sha256: sha256Of([]) });
+            expect(session.putPart).not.toHaveBeenCalled();
+        });
+        it('accounts for exactly what seal adds to a frame', () => {
+            expect((0, archive_seal_util_1.seal)(Buffer.alloc(1234), newKey()).length).toBe(1234 + archive_stream_service_1.SEAL_OVERHEAD_BYTES);
+        });
+    });
+    describe('decodeArchiveFrame', () => {
+        it('refuses a tampered part instead of returning rows from it', async () => {
+            const key = newKey();
+            const session = recordingSession();
+            await (0, archive_stream_service_1.writeArchiveStream)(inBatches(makeRows(100), 100), {
+                key,
+                session,
+                timeColumn: TIME_COLUMN,
+                compressionLevel: COMPRESSION_LEVEL,
+                partMaxBytes: 16 * 1024,
+            });
+            const tampered = Buffer.from(session.payloads.get(1));
+            tampered[tampered.length - 1] ^= 0xff;
+            await expect(decodeAll(tampered, key)).rejects.toThrow();
+        });
+    });
+    describe('splitArchiveFrames', () => {
+        const writeObject = async () => {
+            const key = newKey();
+            const session = recordingSession();
+            const result = await (0, archive_stream_service_1.writeArchiveStream)(inBatches(makeRows(5_000), 1000), {
+                key,
+                session,
+                timeColumn: TIME_COLUMN,
+                compressionLevel: COMPRESSION_LEVEL,
+                partMaxBytes: 4 * 1024,
+            });
+            return { key, parts: result.parts, stored: payloadsInOrder(session, result.parts) };
+        };
+        const streamed = (bytes, chunkBytes = 333) => {
+            const chunks = [];
+            for (let offset = 0; offset < bytes.length; offset += chunkBytes) {
+                chunks.push(bytes.subarray(offset, offset + chunkBytes));
+            }
+            return stream_1.Readable.from(chunks);
+        };
+        const drain = async (source, parts) => {
+            const frames = [];
+            for await (const frame of (0, archive_stream_service_1.splitArchiveFrames)(source, parts)) {
+                frames.push(frame.sealed);
+            }
+            return frames;
+        };
+        it('cuts an object streamed in arbitrary chunks back into its parts', async () => {
+            const { key, parts, stored } = await writeObject();
+            expect(parts.length).toBeGreaterThanOrEqual(3);
+            const frames = await drain(streamed(Buffer.concat(stored)), parts);
+            expect(frames).toEqual(stored);
+            const decoded = (await Promise.all(frames.map((frame) => decodeAll(frame, key)))).flat();
+            expect(decoded).toHaveLength(5_000);
+        });
+        it('refuses an object shorter than its index', async () => {
+            const { parts, stored } = await writeObject();
+            const truncated = Buffer.concat(stored).subarray(0, -10);
+            await expect(drain(streamed(truncated), parts)).rejects.toThrow(/ends inside part/);
+        });
+        it('refuses an object longer than its index', async () => {
+            const { parts, stored } = await writeObject();
+            const extended = Buffer.concat([...stored, Buffer.from('trailing bytes')]);
+            await expect(drain(streamed(extended), parts)).rejects.toThrow(/longer than its/);
+        });
+    });
+});
+//# sourceMappingURL=archive-stream.service.spec.js.map

package/dist/server/archive/tests/archive-stream.service.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"archive-stream.service.spec.js","sourceRoot":"","sources":["../../../../src/server/archive/tests/archive-stream.service.spec.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+CAAiC;AACjC,mCAAkC;AAClC,4DAA4C;AAC5C,sEAOmC;AAWnC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAE5B,MAAM,WAAW,GAAG,WAAW,CAAC;AAChC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;AAExC,MAAM,MAAM,GAAG,GAAW,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;AAEpD,MAAM,gBAAgB,GAAG,GAAG,EAAE;IAC5B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC3C,OAAO;QACL,QAAQ;QACR,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,UAAkB,EAAE,OAAe,EAAE,EAAE;YAC7D,QAAQ,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;QACjD,CAAC,CAAC;KACH,CAAC;AACJ,CAAC,CAAC;AAGF,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,MAAc,EAAU,EAAE;IACrD,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,MAAM,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;QACzD,IAAI,IAAI,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,IAAI,IAAI,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACpF,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;AAC/B,CAAC,CAAC;AAEF,MAAM,QAAQ,GAAG,CACf,KAAa,EACb,gBAA2C,GAAG,EAAE,CAAC,EAAE,EACrC,EAAE,CAChB,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;IAC3C,EAAE,EAAE,OAAO,KAAK,EAAE;IAClB,aAAa,EAAE,wBAAwB;IACvC,KAAK,EAAE,KAAK,GAAG,CAAC;IAChB,OAAO,EAAE,WAAW,KAAK,IAAI,KAAK,CAAC,KAAK,EAAE,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE;IACjE,SAAS,EAAE,IAAI,IAAI,CAAC,SAAS,GAAG,KAAK,GAAG,IAAI,CAAC;CAC9C,CAAC,CAAC,CAAC;AAEN,KAAK,SAAS,CAAC,CAAC,SAAS,CAAC,IAAkB,EAAE,IAAY;IACxD,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,IAAI,EAAE,CAAC;QACvD,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC,CAAC;IACxC,CAAC;AACH,CAAC;AAGD,MAAM,UAAU,GAAG,CAAC,IAAkB,EAAgB,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;AAE1F,MAAM,SAAS,GAAG,KAAK,EAAE,MAAc,EAAE,GAAW,EAAyB,EAAE;IAC7E,MAAM,IAAI,GAAiB,EAAE,CAAC;IAC9B,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAA,2CAAkB,EAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;QACxD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC,CAAC;AAEF,MAAM,QAAQ,GAAG,CAAC,OAAiB,EAAU,EAAE,CAC7C,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAE3E,MAAM,eAAe,GAAG,CACtB,OAA4C,EAC5C,KAAoB,EACV,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAE,CAAC,CAAC;AAE3E,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;IACtC,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,gGAAgG,EAAE,KAAK,IAAI,EAAE;YAC9G,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC;YACrB,MAAM,OAAO,GAAG,gBAAgB,EAAE,CAAC;YACnC,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC9B,MAAM,YAAY,GAAG,EAAE,GAAG,IAAI,CAAC;YAE/B,MAAM,MAAM,GAAG,MAAM,IAAA,2CAAkB,EAAC,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE;gBAC7D,GAAG;gBACH,OAAO;gBACP,UAAU,EAAE,WAAW;gBACvB,gBAAgB,EAAE,iBAAiB;gBACnC,YAAY;aACb,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;YACtD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CACzD,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC,CAC1C,CAAC;YACF,MAAM,CAAC,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;YAE1F,MAAM,OAAO,GAAiB,EAAE,CAAC;YACjC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBAChC,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAE,CAAC;gBAItD,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAE9C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC;gBACxD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBACvC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;gBAC5C,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;gBACxD,OAAO,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;YAC5B,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;YAC1C,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC3C,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;YACtD,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;YAE3F,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAC5D,MAAM,SAAS,GAAG,GAAG,CAAC;YACtB,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,IAAI,OAAO,GAAG,CAAC,CAAC;YAChB,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,MAAM,OAAO,GAAG,GAAS,EAAE;gBACzB,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,GAAG,OAAO,CAAC,CAAC;YACvD,CAAC,CAAC;YAMF,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;gBAC1C,GAAG,GAAG;gBACN,MAAM,EAAE,GAAG,EAAE;oBACX,OAAO,IAAI,CAAC,CAAC;oBACb,OAAO,EAAE,CAAC;oBACV,OAAO,GAAG,CAAC;gBACb,CAAC;aACF,CAAC,CAAC,CAAC;YAEJ,KAAK,SAAS,CAAC,CAAC,YAAY;gBAC1B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,SAAS,EAAE,CAAC;oBAC5D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,SAAS,CAAC,CAAC;oBACnD,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC;oBAC1B,OAAO,EAAE,CAAC;oBACV,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC;YAED,MAAM,MAAM,GAAG,MAAM,IAAA,2CAAkB,EAAC,YAAY,EAAE,EAAE;gBACtD,GAAG,EAAE,MAAM,EAAE;gBACb,OAAO,EAAE,gBAAgB,EAAE;gBAC3B,UAAU,EAAE,WAAW;gBACvB,gBAAgB,EAAE,iBAAiB;gBACnC,YAAY,EAAE,EAAE,GAAG,IAAI;aACxB,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC3C,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAClC,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;YACpE,MAAM,OAAO,GAAG,gBAAgB,EAAE,CAAC;YACnC,MAAM,YAAY,GAAG,EAAE,GAAG,IAAI,CAAC;YAG/B,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;YAEtE,MAAM,MAAM,GAAG,MAAM,IAAA,2CAAkB,EAAC,SAAS,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE;gBAC5D,GAAG,EAAE,MAAM,EAAE;gBACb,OAAO;gBACP,UAAU,EAAE,WAAW;gBACvB,gBAAgB,EAAE,iBAAiB;gBACnC,YAAY;aACb,CAAC,CAAC;YAEH,MAAM,KAAK,GAAG,eAAe,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACpF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,MAAM,CAAC,IAAI,CAAC,CAAC,mBAAmB,CAAC,YAAY,CAAC,CAAC;YACjD,CAAC;YAED,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACtC,MAAM,CAAC,IAAI,CAAC,CAAC,eAAe,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;YACpD,CAAC;YACD,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QACrF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,+EAA+E,EAAE,KAAK,IAAI,EAAE;YAC7F,MAAM,OAAO,GAAG,gBAAgB,EAAE,CAAC;YAEnC,MAAM,MAAM,CACV,IAAA,2CAAkB,EAChB,SAAS,CACP,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,EACxB,CAAC,CACF,EACD;gBACE,GAAG,EAAE,MAAM,EAAE;gBACb,OAAO;gBACP,UAAU,EAAE,WAAW;gBACvB,gBAAgB,EAAE,iBAAiB;gBACnC,YAAY,EAAE,IAAI;aACnB,CACF,CACF,CAAC,OAAO,CAAC,OAAO,CAAC,+BAA+B,CAAC,CAAC;YACnD,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2EAA2E,EAAE,KAAK,IAAI,EAAE;YACzF,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,QAAQ,CAAC,EAAE,CAAC,CAAC;YAE9C,MAAM,MAAM,CACV,IAAA,2CAAkB,EAAC,SAAS,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,EAAE,EAAE,CAAC,EAAE;gBAC1D,GAAG,EAAE,MAAM,EAAE;gBACb,OAAO,EAAE,gBAAgB,EAAE;gBAC3B,UAAU,EAAE,WAAW;gBACvB,gBAAgB,EAAE,iBAAiB;gBACnC,YAAY,EAAE,EAAE,GAAG,IAAI;aACxB,CAAC,CACH,CAAC,OAAO,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kFAAkF,EAAE,KAAK,IAAI,EAAE;YAChG,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC;YACrB,MAAM,OAAO,GAAG,gBAAgB,EAAE,CAAC;YAEnC,MAAM,IAAI,GAAiB,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;gBACtE,EAAE,EAAE,WAAW,KAAK,EAAE;gBACtB,qBAAqB,EAAE,sBAAsB;gBAC7C,SAAS,EAAE,IAAI,IAAI,CAAC,SAAS,GAAG,KAAK,GAAG,IAAI,CAAC;gBAC7C,YAAY,EAAE,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;aAC/B,CAAC,CAAC,CAAC;YAEJ,MAAM,MAAM,GAAG,MAAM,IAAA,2CAAkB,EAAC,SAAS,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE;gBAC5D,GAAG;gBACH,OAAO;gBACP,UAAU,EAAE,WAAW;gBACvB,gBAAgB,EAAE,iBAAiB;gBACnC,YAAY,EAAE,CAAC,GAAG,IAAI;aACvB,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;YACtD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;gBAChC,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAE,EAAE,GAAG,CAAC,CAAC;gBAC9E,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YAC1D,CAAC;YAED,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;YACtC,MAAM,MAAM,CACV,IAAA,2CAAkB,EAAC,SAAS,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE;gBAC3D,GAAG;gBACH,OAAO,EAAE,gBAAgB,EAAE;gBAC3B,UAAU,EAAE,WAAW;gBACvB,gBAAgB,EAAE,iBAAiB;gBACnC,YAAY,EAAE,CAAC,GAAG,IAAI;aACvB,CAAC,CACH,CAAC,OAAO,CAAC,OAAO,CAAC,sBAAsB,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,2EAA2E,EAAE,KAAK,IAAI,EAAE;YACzF,MAAM,MAAM,CACV,IAAA,2CAAkB,EAAC,SAAS,CAAC,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE;gBACvE,GAAG,EAAE,MAAM,EAAE;gBACb,OAAO,EAAE,gBAAgB,EAAE;gBAC3B,UAAU,EAAE,WAAW;gBACvB,gBAAgB,EAAE,iBAAiB;gBACnC,YAAY,EAAE,EAAE,GAAG,IAAI;aACxB,CAAC,CACH,CAAC,OAAO,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACrD,MAAM,OAAO,GAAG,gBAAgB,EAAE,CAAC;YAEnC,MAAM,MAAM,GAAG,MAAM,IAAA,2CAAkB,EAAC,SAAS,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE;gBACzD,GAAG,EAAE,MAAM,EAAE;gBACb,OAAO;gBACP,UAAU,EAAE,WAAW;gBACvB,gBAAgB,EAAE,iBAAiB;gBACnC,YAAY,EAAE,EAAE,GAAG,IAAI;aACxB,CAAC,CAAC;YAEH,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YACzF,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC;QACjD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;YAGxD,MAAM,CAAC,IAAA,wBAAI,EAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,GAAG,4CAAmB,CAAC,CAAC;QACrF,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;YACzE,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC;YACrB,MAAM,OAAO,GAAG,gBAAgB,EAAE,CAAC;YACnC,MAAM,IAAA,2CAAkB,EAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,EAAE;gBACtD,GAAG;gBACH,OAAO;gBACP,UAAU,EAAE,WAAW;gBACvB,gBAAgB,EAAE,iBAAiB;gBACnC,YAAY,EAAE,EAAE,GAAG,IAAI;aACxB,CAAC,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAC;YACvD,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC;YAEtC,MAAM,MAAM,CAAC,SAAS,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3D,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,MAAM,WAAW,GAAG,KAAK,IAAI,EAAE;YAC7B,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC;YACrB,MAAM,OAAO,GAAG,gBAAgB,EAAE,CAAC;YACnC,MAAM,MAAM,GAAG,MAAM,IAAA,2CAAkB,EAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,EAAE;gBACxE,GAAG;gBACH,OAAO;gBACP,UAAU,EAAE,WAAW;gBACvB,gBAAgB,EAAE,iBAAiB;gBACnC,YAAY,EAAE,CAAC,GAAG,IAAI;aACvB,CAAC,CAAC;YACH,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,eAAe,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QACtF,CAAC,CAAC;QAGF,MAAM,QAAQ,GAAG,CAAC,KAAa,EAAE,UAAU,GAAG,GAAG,EAAY,EAAE;YAC7D,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,MAAM,IAAI,UAAU,EAAE,CAAC;gBACjE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC;YAC3D,CAAC;YACD,OAAO,iBAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC/B,CAAC,CAAC;QAEF,MAAM,KAAK,GAAG,KAAK,EAAE,MAAgB,EAAE,KAAoB,EAAqB,EAAE;YAChF,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAA,2CAAkB,EAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC;gBAC5D,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC5B,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC;QAEF,EAAE,CAAC,iEAAiE,EAAE,KAAK,IAAI,EAAE;YAC/E,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,WAAW,EAAE,CAAC;YACnD,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;YAE/C,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YAEnE,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAC/B,MAAM,OAAO,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACzF,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACxD,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,WAAW,EAAE,CAAC;YAC9C,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YAEzD,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;QACtF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,WAAW,EAAE,CAAC;YAC9C,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAE3E,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;QACpF,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/server/archive/zstd.util.d.ts

@@ -0,0 +1,19 @@
+import { Transform } from 'stream';
+export interface ZstdOptions {
+    params: Record<number, number>;
+}
+export interface ZstdCompressStream extends Transform {
+    flush(callback: (error?: Error | null) => void): void;
+}
+interface ZstdBindings {
+    createZstdCompress(options: ZstdOptions): ZstdCompressStream;
+    createZstdDecompress(): Transform;
+    zstdCompressSync(buffer: Buffer, options: ZstdOptions): Buffer;
+    zstdDecompressSync(buffer: Buffer): Buffer;
+    constants: {
+        ZSTD_c_compressionLevel: number;
+    };
+}
+export declare const zstd: ZstdBindings;
+export declare const zstdCompressionOptions: (compressionLevel: number) => ZstdOptions;
+export {};

package/dist/server/archive/zstd.util.js

@@ -0,0 +1,43 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.zstdCompressionOptions = exports.zstd = void 0;
+const zlib = __importStar(require("zlib"));
+exports.zstd = zlib;
+const zstdCompressionOptions = (compressionLevel) => ({
+    params: { [exports.zstd.constants.ZSTD_c_compressionLevel]: compressionLevel },
+});
+exports.zstdCompressionOptions = zstdCompressionOptions;
+//# sourceMappingURL=zstd.util.js.map

package/dist/server/archive/zstd.util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"zstd.util.js","sourceRoot":"","sources":["../../../src/server/archive/zstd.util.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,2CAA6B;AAiChB,QAAA,IAAI,GAAG,IAA+B,CAAC;AAG7C,MAAM,sBAAsB,GAAG,CAAC,gBAAwB,EAAe,EAAE,CAAC,CAAC;IAChF,MAAM,EAAE,EAAE,CAAC,YAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC,EAAE,gBAAgB,EAAE;CACvE,CAAC,CAAC;AAFU,QAAA,sBAAsB,0BAEhC"}

package/dist/server/index.d.ts

@@ -6,3 +6,4 @@ export * from './explore/field-catalog.types';
 export * from './explore/sql-translator';
 export * from './explore/like-escape';
 export * from './abyss-signature.util';
+export * from './archive';

package/dist/server/index.js

@@ -22,4 +22,5 @@ __exportStar(require("./explore/field-catalog.types"), exports);
 __exportStar(require("./explore/sql-translator"), exports);
 __exportStar(require("./explore/like-escape"), exports);
 __exportStar(require("./abyss-signature.util"), exports);
+__exportStar(require("./archive"), exports);
 //# sourceMappingURL=index.js.map

package/dist/server/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/server/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAgBA,kDAAgC;AAGhC,2DAAyC;AAKzC,wDAAsC;AAGtC,yCAAuB;AAIvB,gEAA8C;AAC9C,2DAAyC;AACzC,wDAAsC;AAGtC,yDAAuC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/server/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAgBA,kDAAgC;AAGhC,2DAAyC;AAKzC,wDAAsC;AAGtC,yCAAuB;AAIvB,gEAA8C;AAC9C,2DAAyC;AACzC,wDAAsC;AAGtC,yDAAuC;AAKvC,4CAA0B"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@abyss-project/main",
-  "version": "34.1.2",
+  "version": "34.2.0",
   "description": "Core package to interact with Abyss-Project",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",