@harperfast/harper 5.2.10 → 5.3.0-alpha.1

package/dist/resources/blob.d.ts

@@ -10,7 +10,8 @@
  *   - While the file is being written, 0xffffffffffff is used as a placeholder to indicate that the file is not finished being written (this nicely matches the logic that if the written content size is less than the indicated content size, it is not finished)
  *   - Note that for compressed data, the size is the uncompressed size, and the compressed size in the file
  */
-import { Readable } from 'node:stream';
+import { watch, type FSWatcher } from 'node:fs';
+import { Readable, Transform } from 'node:stream';
 import type { RootDatabase } from 'lmdb';
 type StorageInfo = {
     storageIndex: number;
@@ -18,9 +19,10 @@ type StorageInfo = {
     store?: any;
     filePath?: string;
     recordId?: number;
-    contentBuffer?: any;
+    contentBuffer?: Buffer;
     source?: Readable;
     compress?: boolean;
+    storedCodec?: 'deflate';
     flush?: boolean;
     start?: number;
     end?: number;
@@ -28,7 +30,15 @@ type StorageInfo = {
     saved?: boolean;
     asString?: string;
     deleteOnFailure?: boolean;
+    fileState?: {
+        discarded?: boolean;
+    };
+    probedDamage?: {
+        fileSize: number;
+        header: Buffer;
+    };
 };
+export declare const DEFLATE_TYPE = 1;
 export declare function registerBlobReceiveInFlight(fileId: string | null | undefined, store: any): void;
 export declare function unregisterBlobReceiveInFlight(fileId: string | null | undefined, store: any): void;
 export declare function isBlobReceiveInFlight(fileId: string | null | undefined, store: any): boolean;
@@ -73,6 +83,22 @@ export declare function createPendingMarkerBarrier(options: {
     onWriteError?: (writeError: unknown) => void;
 }): (writeError?: unknown) => void;
 export declare const BLOB_UNAVAILABLE_STATUS = 503;
+/**
+ * Arm the one-shot watch that wakes an in-progress blob read of `filePath`, or return `undefined`
+ * for a read that has to poll instead. `isLive` decides whether the read still owns a callback's
+ * watcher: a callback from one it has already replaced must not act, or it would close the live
+ * watcher and start a second read at the same position. `onFailure` runs only for a live watcher
+ * that fails after registration — a registration that throws latches `mustPoll` and leaves the
+ * caller in its own no-watcher branch.
+ */
+export declare function watchInProgressFile(filePath: string, watchTarget: {
+    path: string;
+    mustPoll: boolean;
+}, handlers: {
+    isLive: (watcher: FSWatcher) => boolean;
+    onChange: () => void;
+    onFailure: () => void;
+}, watchFile?: typeof watch): FSWatcher | undefined;
 declare const FileBackedBlob_base: {
     new (): Blob;
 };
@@ -140,14 +166,44 @@ export type BlobCreationOptions = {
     size?: number;
     saveBeforeCommit?: boolean;
 };
+/**
+ * Whether a new blob of `type` and (uncompressed) `size` should be deflate-compressed on disk,
+ * per the opt-in `storage.blobs.compression` map. Absent config means never. Matching precedence
+ * is exact type, then `type/*`, then `default`. An unknown size never compresses: the threshold
+ * cannot be evaluated, and compressing a small streamed write under a large configured threshold
+ * would violate the operator's limit.
+ */
+export declare function resolveBlobCompression(type: string | undefined, size: number | undefined): boolean;
 /**
  * Create a blob from a readable stream or a buffer by creating a file in the blob storage path with a new unique internal id, that
  * can be saved/stored.
  * @param source
  */
 export declare function createBlob(source: Uint8Array | NodeJS.ReadableStream | string | Iterable<Uint8Array> | AsyncIterator<Uint8Array>, options?: BlobCreationOptions): Blob;
+/**
+ * Create a blob whose source stream carries the STORED representation — an already-deflated body —
+ * rather than content bytes, so a replication receiver can land a peer's compressed blob without
+ * recompressing it. `size` is the uncompressed content length (it goes in the header; a raw
+ * compressed source cannot derive it) and the body is verified on write: it must be a well-formed
+ * deflate stream inflating to exactly `size` bytes, or the save rejects (see
+ * createStoredDeflateVerifier). Internal to replication on purpose — not part of BlobCreationOptions
+ * and not exported to components.
+ */
+export declare function createBlobFromStoredBody(source: NodeJS.ReadableStream, options: {
+    type?: string;
+    size: number;
+    codec: 'deflate';
+}): Blob;
 export declare function saveBlob(blob: FileBackedBlob, deleteOnFailure?: boolean): StorageInfo;
 export declare function shouldDestroyIdleBlobSource(paused: boolean, bytesWritten: number, lastProgressBytes: number): boolean;
+/**
+ * Pass-through for a pre-compressed (stored deflate) blob body that concurrently inflates what it
+ * forwards and fails the pipeline unless the body is a well-formed deflate stream inflating to
+ * exactly `expectedSize` bytes. This is what lets a raw peer-preserved body land under a finalized
+ * DEFLATE header without trusting the peer: a truncated, corrupt, oversized (bomb), or trailing-
+ * garbage body rejects the save instead of being published.
+ */
+export declare function createStoredDeflateVerifier(expectedSize: number, fileId: string): Transform;
 export declare function getFileId(blob: Blob): string;
 export declare function isSaving(blob: Blob): Promise<void>;
 export declare function getFilePathForBlob(blob: FileBackedBlob): string;
@@ -155,17 +211,49 @@ export declare function getFilePathForBlob(blob: FileBackedBlob): string;
  * Repairs a damaged file-backed blob under its existing fileId. The damage check and atomic file
  * replacement share the blob lock, so a failed repair never modifies the referenced file. Callers
  * must establish an exact record identity tie (same version and source node) and positional blob
- * pairing before calling. The received byte count must match the reported source size and, when
- * present, the stored descriptor. On a synchronous `undefined`, no repair started and `source`
- * remains owned by the caller.
+ * pairing before calling, and classify the file with `blobFileMissingOrIncompleteAsync` first — a
+ * compressed body's damage is only provable by inflating it, which the locked recheck here will
+ * not do. The received byte count must match the reported source size and, when present, the
+ * stored descriptor. On a synchronous `undefined`, no repair started and `source` remains owned
+ * by the caller.
  */
 export declare function repairBlobFile(blob: Blob, source: Readable, sourceSize?: number | (() => number | undefined) | undefined): Promise<void> | undefined;
+export declare function isSystemicIoError(error: unknown): boolean;
+/**
+ * How a consumer that captures a blob root (backup snapshot, backup archive) should treat one file.
+ * `skip` is not a blob to capture at all; `pending` and `gone` require markers; `capture` is taken as-is.
+ * `capture` means settled as far as the path can show, which is short of a guarantee: a known-size write
+ * that has landed every byte is indistinguishable here from a finished one, and if it then aborts, the
+ * PENDING stamp rewrites that inode in place, truncating any same-filesystem hard link taken from it.
+ * Telling the two apart needs the blob write lock, which is keyed by file id and unreachable from a walk.
+ */
+export declare function classifyBlobFileForCapture(filePath: string): Promise<BlobCaptureDisposition>;
+/**
+ * What a consumer capturing a blob root should do with one file. `pending` is a blob that was not
+ * whole *yet* (retryable 503); `gone` is absent from this capture and represented as terminal 500.
+ * Both still put a file at the id, because `getNextFileId` recovers the counter by scanning the
+ * directory and an absent file lets a restored record's id be reissued.
+ */
+export type BlobCaptureDisposition = 'skip' | 'capture' | 'pending' | 'gone';
+/** The stand-in bytes for a blob that could not be captured whole. */
+export declare function createCaptureMarker(disposition: 'pending' | 'gone', message: string): Buffer;
 export declare function blobHeaderIndicatesIncomplete(header: Buffer, fileSize: number): boolean;
 /**
- * Whether a file-backed blob's backing file is missing or incomplete on disk — the gate for the
- * copy-delivery repair. This blocking probe is the locked final recheck after the caller's async
- * exact-duplicate prefilter. Compressed bodies require the asynchronous repair sweep for
- * verification. Returns undefined for blobs the question does not apply to.
+ * Whether a file-backed blob's backing file is missing or incomplete on disk — the unlocked,
+ * non-blocking classification a copy-delivery repair candidate gets first (harper-pro#699). A deflate
+ * body is inflated (streamed, off the event loop) to decide; the file it judged damaged is remembered
+ * so the locked recheck in `repairBlobFile` can recognize it without inflating again. Returns
+ * undefined for blobs the question does not apply to, or that could not be inspected.
+ */
+export declare function blobFileMissingOrIncompleteAsync(blob: Blob): Promise<boolean | undefined>;
+/**
+ * The locked, synchronous recheck of `blobFileMissingOrIncompleteAsync`'s verdict, run by
+ * `repairBlobFile` under the blob's write lock so a repair can only replace a file no writer is
+ * still producing. It repeats the header classification, and for a deflate body — which it cannot
+ * afford to inflate on the event loop — answers `true` only while the file is still the one the
+ * probe found damaged: a body can only grow (writers append; a repair replaces the file) so an
+ * unchanged length and header mean unchanged bytes, and anything else is `undefined` (not provably
+ * damaged — decline, and let the next probe or the repair sweep classify it).
  */
 export declare function blobFileMissingOrIncomplete(blob: Blob): boolean | undefined;
 export declare const databasePaths: Map<RootDatabase<any, import("lmdb").Key>, string[]>;
@@ -294,6 +382,14 @@ export declare function collectRetainedFileIds(record: any): Set<string> | undef
  * @param database
  */
 export declare function cleanupOrphans(database: any, databaseName?: string): Promise<number>;
+/**
+ * Whether the deflate body after the header inflates to exactly `size` bytes. A compressed blob's
+ * header records the uncompressed length, so it cannot be compared to the on-disk body; a truncated
+ * stream errors (Z_BUF_ERROR) or yields fewer bytes. Streamed rather than inflateSync so memory stays
+ * bounded over a sweep that may touch many large blobs, and abandoned as soon as the output passes
+ * `size` so a body that lies about its length costs no more than the length it claimed.
+ */
+export declare function inflatesToExactly(filePath: string, size: number): Promise<boolean>;
 /**
  * Resolves true if the given blob's backing file is present and complete: no error stub, not an
  * in-flight placeholder, and the content matches the header (uncompressed blobs by body length,
@@ -301,6 +397,30 @@ export declare function cleanupOrphans(database: any, databaseName?: string): Pr
  * written after a peer fetch.
  */
 export declare function isBlobComplete(blob: Blob): Promise<boolean>;
+export type StoredBlobBody = {
+    codec: 'deflate';
+    size: number;
+    /** The raw post-header bytes. Single-use; releases the file on completion, error, or teardown. */
+    stream: () => AsyncGenerator<Buffer>;
+    /** Release the descriptor and file hold without streaming. */
+    close: () => void;
+};
+/**
+ * Open a blob's STORED body for transfer, without inflating it — the sender half of replication's
+ * codec preservation. Returns undefined whenever the stored form is not a settled deflate file
+ * (uncompressed, sliced, still being written, missing, or already condemned); the caller then falls
+ * back to the ordinary content stream, which knows how to wait and how to report errors.
+ *
+ * Deliberately synchronous through the decision: the caller decides raw-vs-inflated inside the same
+ * synchronous block that puts the owning record frame on the wire. The stream opens its own
+ * descriptor later and re-reads the header first: a settled file's header never changes except by
+ * a repair publishing a fresh (uncompressed) file over the path, so a header that differs from the
+ * sniffed one means the announced form is gone — reported as transient (503), since a retry
+ * re-decides against the new file. The stream verifies as it sends — a concurrent inflate must
+ * land on exactly `size` bytes — so a torn body that survived the header checks becomes a loud,
+ * permanently-classified error instead of replicated garbage.
+ */
+export declare function openStoredBlobBody(blob: Blob): StoredBlobBody | undefined;
 /**
  * Async generator that yields records whose referenced blob files are missing, truncated, or in an
  * error state. Used by the blob repair sweep to find candidates for peer-fetched recovery.