@hostr/upload 0.1.0

package/dist/core/engine.d.ts

@@ -0,0 +1,150 @@
+import type { Clock, FileSource, HttpTransport, StateStore, NetworkMonitor, Hasher } from './types.js';
+import { type SessionRecord, type SessionStatus } from './session.js';
+import { type Progress } from './events.js';
+import { type HostrApi } from '../api/client.js';
+export type CompleteInfo = {
+    assetId: string;
+    etag: string;
+    sha256?: string;
+    bytes: number;
+};
+export type EngineDeps = {
+    api: HostrApi;
+    file: FileSource;
+    transport: HttpTransport;
+    store: StateStore;
+    hasher: Hasher;
+    clock: Clock;
+    network?: NetworkMonitor;
+    rng?: () => number;
+};
+export type EngineOptions = {
+    concurrency: number;
+    checksum: 'sha256' | 'none';
+    presignBatch: number;
+    presignTtlSec: number;
+    onTokenExpired?: (assetId: string) => Promise<string>;
+};
+/** Advisory signal that every part PUT is dying at the network layer while
+ *  zero bytes have EVER been confirmed for the session. To XHR a CORS-blocked
+ *  PUT is indistinguishable from network loss, so without this the two cases
+ *  present identically (running↔stalled forever). Zero-ever-confirmed is the
+ *  discriminator from mid-upload connectivity loss; a genuinely-offline client
+ *  usually never gets here at all (API calls stall first, or a NetworkMonitor
+ *  parks dispatch before any PUT is attempted). Never terminal — a long
+ *  offline stretch from byte zero looks the same and must still recover — the
+ *  listener decides what to do with it. */
+export type TransportDiagnostic = {
+    code: 'systemic-transport-failure';
+    consecutiveNetworkFailures: number;
+    bytesEverConfirmed: number;
+    hint: string;
+};
+/** Consecutive network-class part-PUT failures (with zero bytes ever
+ *  confirmed) before a 'diagnostic' event fires. */
+export declare const SYSTEMIC_TRANSPORT_FAILURE_THRESHOLD = 3;
+export type EngineEvents = {
+    progress: Progress;
+    statechange: SessionStatus;
+    error: Error;
+    complete: CompleteInfo;
+    diagnostic: TransportDiagnostic;
+};
+export declare class IntegrityError extends Error {
+    constructor(message?: string);
+}
+export declare class UploadEngine {
+    readonly done: Promise<CompleteInfo>;
+    private resolveDone;
+    private rejectDone;
+    private emitter;
+    private presigned;
+    /** Presign fetches since the last confirmed part. Healthy uploads confirm
+     *  constantly, so this stays at zero and costs nothing; in a sustained
+     *  failure loop (every PUT dying without landing a part) it grows, and
+     *  presign() spaces further /parts fetches on the same backoff curve the
+     *  'expired' storm uses — the API call rate is what trips rate limits, so
+     *  it gets its own governor, independent of per-part attempt cadence and
+     *  worker count. */
+    private presignsSinceConfirm;
+    /** The in-flight /parts fetch, shared so concurrent cache-missing workers
+     *  ride one round-trip and re-check the cache instead of each firing an
+     *  overlapping batch of their own (same memo shape as hashLanePromise). */
+    private presignInflight;
+    private claimed;
+    private inflightSent;
+    private stopped;
+    private running;
+    private restartPending;
+    private startedAt;
+    private confirmedAtStart;
+    private lastEmit;
+    private deps;
+    private opts;
+    private session;
+    /** The in-flight sha256 lane, tracked so a pause→resume never starts a
+     *  second concurrent lane over the same file. Cleared once settled so a
+     *  later resume can start a fresh one from the persisted midstate. */
+    private hashLanePromise;
+    /** Engine-wide (not per-part): a CORS-blocked bucket fails every worker's
+     *  PUT the same way, so the streak accumulates across parts. Deliberately
+     *  not bumped by apiCall's network stalls — the API answering while every
+     *  R2 PUT dies is exactly the bucket-CORS signature. */
+    private consecutiveNetworkFailures;
+    private diagnosticFired;
+    /** Resolvers for anyone currently parked in waitOnline(), woken the
+     *  instant pause()/abort()/fail() halts the engine — without this a
+     *  stalled engine could only be interrupted by the network itself
+     *  coming back online. */
+    private stopWaiters;
+    constructor(deps: EngineDeps, opts: EngineOptions, session: SessionRecord);
+    on<K extends keyof EngineEvents>(ev: K, cb: (v: EngineEvents[K]) => void): () => void;
+    get state(): SessionStatus;
+    start(): void;
+    pause(): void;
+    resume(): void;
+    abort(): Promise<void>;
+    /** Stop dispatch everywhere at once: new part claims, waitOnline's
+     *  parked callers, and apiCall's network-retry loop all observe this on
+     *  their next check. */
+    private halt;
+    private setStatus;
+    private fail;
+    private run;
+    private finish;
+    private expectedComposite;
+    /** Reconcile local bookkeeping against what R2 actually holds before resuming.
+     *  Local claims R2 can't confirm are stale and forgotten; parts whose stored
+     *  md5 matches R2's etag are kept as-is; a mismatch means the landed bytes
+     *  aren't verified as ours, so it's dropped for re-upload. For parts R2 holds
+     *  but local `done` has no record of (state lost — reinstall, cleared
+     *  storage), the local byte range is re-read and re-hashed; it's only
+     *  adopted as done when that hash matches R2's etag, never trusted blind. */
+    private reconcile;
+    private nextPart;
+    private worker;
+    private uploadPart;
+    private presign;
+    /** Fresh = safely usable for a new PUT: at least 60s of validity left, so
+     *  a part starting now won't have its URL expire mid-flight. */
+    private presignFresh;
+    /** API calls: infinite patience for network loss, one token refresh on 401, else throw.
+     *  Throws Stopped (never surfaced) if pause()/abort() interrupts a stuck retry. */
+    private apiCall;
+    private waitOnline;
+    /** Lazily starts the sha256 lane and remembers the in-flight promise so a
+     *  fast pause→resume can never spawn a second concurrent lane over the
+     *  same file. The promise carries its own no-op .catch so it can never
+     *  become an unhandledRejection while nothing is actively awaiting it
+     *  (e.g. during a paused window); a real failure still surfaces normally
+     *  to whoever does await it, and from there through run() to fail(). */
+    private getHashLane;
+    private hashLane;
+    /** Feed every part-PUT outcome to the systemic-transport-failure detector.
+     *  Any outcome that isn't network-class — success, 403, 5xx, even corrupt
+     *  bytes — proves a request reached the server, so the streak resets. */
+    private notePutOutcome;
+    private bytesConfirmed;
+    private onPartProgress;
+    private emitProgress;
+}

package/dist/core/events.d.ts

@@ -0,0 +1,33 @@
+/** Minimal typed pub/sub. A throwing listener is isolated: it never
+ *  prevents emit from reaching later listeners, and never propagates to
+ *  the caller of emit. */
+export declare class Emitter<E extends Record<string, unknown>> {
+    private listeners;
+    on<K extends keyof E>(ev: K, cb: (v: E[K]) => void): () => void;
+    emit<K extends keyof E>(ev: K, v: E[K]): void;
+}
+export type Progress = {
+    bytesSent: number;
+    bytesConfirmed: number;
+    totalBytes: number;
+    throughputBps: number | null;
+    etaSec: number | null;
+    partsDone: number;
+    partsTotal: number;
+    /** Optional passthrough for a per-part status grid (Task 23's playground).
+     *  computeProgress does not populate this itself; the engine will. */
+    partStates?: Record<number, 'inflight' | 'done'>;
+};
+export declare function computeProgress(i: {
+    confirmed: number;
+    inflight: number;
+    total: number;
+    partsDone: number;
+    partsTotal: number;
+    startedAtMs: number;
+    nowMs: number;
+    /** Bytes confirmed when this run began (not a timestamp, despite the
+     *  similar-looking field names elsewhere — this is a byte count). */
+    confirmedAtStart: number;
+    partStates?: Record<number, 'inflight' | 'done'>;
+}): Progress;

package/dist/core/retry.d.ts

@@ -0,0 +1,20 @@
+import type { PutResult } from './types.js';
+/**
+ * Design rationale: network-class errors never exhaust a budget —
+ * connectivity loss stalls patiently, since it says nothing about whether
+ * the server-side operation was valid. Finite retry budgets exist only for
+ * unexpected server responses (5xx, 429) where retrying blindly could be
+ * futile or harmful. 'expired' and 'fatal' are not retried at all, just
+ * surfaced for the caller to recover (re-presign) or abort.
+ */
+export type ErrorClass = 'network' | 'transient' | 'expired' | 'corrupt' | 'fatal';
+export declare const RETRY_BUDGET = 6;
+/** Classify the outcome of a PUT (e.g. an R2/S3 presigned upload). */
+export declare function classifyPut(res: PutResult | null, err: unknown): ErrorClass;
+/** Classify the outcome of an API call (create/complete/token endpoints). */
+export declare function classifyApi(err: unknown): ErrorClass;
+/**
+ * Exponential backoff with jitter: min(cap, 500 * 2^(attempt-1) * (1 + 0.5*rng())).
+ * `rng` defaults to Math.random; `capMs` defaults to 60000.
+ */
+export declare function backoffMs(attempt: number, rng?: () => number, capMs?: number): number;

package/dist/core/session.d.ts

@@ -0,0 +1,28 @@
+import type { StateStore } from './types.js';
+export type SessionStatus = 'idle' | 'running' | 'stalled' | 'paused' | 'completing' | 'complete' | 'failed' | 'aborted';
+export type SessionRecord = {
+    v: 1;
+    assetId: string;
+    endpoint: string;
+    uploadToken: string;
+    key: string;
+    partSize: number;
+    expectedParts: number;
+    declaredSize: number;
+    fileId: string;
+    done: Record<number, string>;
+    hash?: {
+        offset: number;
+        state: string;
+    };
+    sha256?: string;
+    status: SessionStatus;
+    createdAt: string;
+};
+export declare const storageKey: (fileId: string) => string;
+export declare function loadSession(store: StateStore, fileId: string): Promise<SessionRecord | null>;
+export declare function saveSession(store: StateStore, s: SessionRecord): Promise<void>;
+export declare function clearSession(store: StateStore, fileId: string): Promise<void>;
+/** partSize except a short final part. */
+export declare function partSizeOf(s: SessionRecord, n: number): number;
+export declare function memoryStore(): StateStore;

package/dist/core/types.d.ts

@@ -0,0 +1,44 @@
+/** Adapter seams. Everything platform-specific enters the engine through these. */
+export interface FileSource {
+    size: number;
+    /** Stable identity across relaunches: name:size:mtime (browser) or path:size:mtime. */
+    id: string;
+    read(offset: number, length: number): Promise<Uint8Array>;
+    close?(): Promise<void>;
+}
+export type PutResult = {
+    status: number;
+    etag: string | null;
+};
+export interface HttpTransport {
+    put(url: string, body: Uint8Array, opts?: {
+        onProgress?: (sentBytes: number) => void;
+        signal?: AbortSignal;
+    }): Promise<PutResult>;
+}
+/** Persisted across kill/relaunch. Values are opaque strings (JSON). */
+export interface StateStore {
+    get(key: string): Promise<string | null>;
+    set(key: string, value: string): Promise<void>;
+    delete(key: string): Promise<void>;
+}
+export interface NetworkMonitor {
+    isOnline(): boolean;
+    onChange(cb: (online: boolean) => void): () => void;
+}
+export interface IncrementalHash {
+    update(data: Uint8Array): void;
+    digestHex(): string;
+    digestBytes(): Uint8Array;
+    exportState?(): string;
+    importState?(state: string): void;
+}
+/** Per-part hashing seam. Node swaps in node:crypto MD5; the SHA-256 lane
+ *  always uses the pure implementation for portable midstate. */
+export interface Hasher {
+    md5(): IncrementalHash;
+}
+export interface Clock {
+    now(): number;
+    sleep(ms: number): Promise<void>;
+}

package/dist/core/upload.d.ts

@@ -0,0 +1,32 @@
+import type { Clock, FileSource, HttpTransport, StateStore, Hasher, NetworkMonitor } from './types.js';
+import { UploadEngine } from './engine.js';
+export type UploadOptions = {
+    endpoint: string;
+    uploadToken: string;
+    assetId: string;
+    file: FileSource;
+    transport: HttpTransport;
+    store: StateStore;
+    hasher: Hasher;
+    network?: NetworkMonitor;
+    clock?: Clock;
+    fetchFn?: typeof fetch;
+    concurrency?: number;
+    checksum?: 'sha256' | 'none';
+    presignTtlSec?: number;
+    onTokenExpired?: (assetId: string) => Promise<string>;
+};
+export type Upload = Pick<UploadEngine, 'done' | 'state' | 'on' | 'start' | 'pause' | 'resume' | 'abort'>;
+/** Resolves (or creates) the persisted session for `file.id`, wires an
+ *  `HostrApi` and `UploadEngine` around it, and returns the engine as the
+ *  public `Upload` surface. A session already saved for this file wins over
+ *  `opts.assetId` — the caller's backend may have minted a fresh upload
+ *  without knowing one was already in flight — but `opts.uploadToken` is
+ *  always adopted, since the saved token may be stale. */
+export declare function buildUpload(opts: UploadOptions): Promise<Upload>;
+/** Lets an app skip minting a fresh upload token when one is already in
+ *  flight for this file — check first, then only call the backend if this
+ *  returns null. */
+export declare function findResumable(store: StateStore, fileId: string): Promise<{
+    assetId: string;
+} | null>;

package/dist/hash/composite.d.ts

@@ -0,0 +1,2 @@
+/** R2's multipart object ETag: MD5 of the concatenated raw part digests, dash, part count. */
+export declare function compositeEtag(md5Hexes: string[]): string;

package/dist/hash/hasher.d.ts

@@ -0,0 +1,2 @@
+import type { Hasher } from '../core/types.js';
+export declare const pureHasher: Hasher;

package/dist/hash/hex.d.ts

@@ -0,0 +1,2 @@
+export declare function bytesToHex(b: Uint8Array): string;
+export declare function hexToBytes(s: string): Uint8Array;

package/dist/hash/md5.d.ts

@@ -0,0 +1,24 @@
+import type { IncrementalHash } from '../core/types.js';
+export declare class Md5 implements IncrementalHash {
+    private h;
+    private buf;
+    private bufLen;
+    private len;
+    update(data: Uint8Array): void;
+    private compress;
+    /**
+     * Finalize on a clone so the hash can keep accepting updates.
+     *
+     * Padding: capture the message length in bits BEFORE touching the clone,
+     * then build the entire 0x80 + zero-padding + 64-bit length tail as one
+     * buffer sized so the clone's running byte count lands on a block
+     * boundary, and hand it to update() in a single call. update()'s own
+     * `len` bookkeeping on the clone is irrelevant after this point (the
+     * clone is discarded), so there is no need to keep it meaningful.
+     */
+    digestBytes(): Uint8Array;
+    digestHex(): string;
+    exportState(): string;
+    importState(state: string): void;
+    private clone;
+}

package/dist/hash/sha256.d.ts

@@ -0,0 +1,25 @@
+import type { IncrementalHash } from '../core/types.js';
+export declare class Sha256 implements IncrementalHash {
+    private h;
+    private buf;
+    private bufLen;
+    private len;
+    private w;
+    update(data: Uint8Array): void;
+    private compress;
+    /**
+     * Finalize on a clone so the hash can keep accepting updates.
+     *
+     * Padding: capture the message length in bits BEFORE touching the clone,
+     * then build the entire 0x80 + zero-padding + 64-bit length tail as one
+     * buffer sized so the clone's running byte count lands on a block
+     * boundary, and hand it to update() in a single call. update()'s own
+     * `len` bookkeeping on the clone is irrelevant after this point (the
+     * clone is discarded), so there is no need to keep it meaningful.
+     */
+    digestBytes(): Uint8Array;
+    digestHex(): string;
+    exportState(): string;
+    importState(state: string): void;
+    private clone;
+}

package/dist/index.d.ts

@@ -0,0 +1,13 @@
+export type * from './core/types.js';
+export { ApiError, NetworkError, HostrApi, createUploadSession, mintUploadToken, } from './api/client.js';
+export type { RemotePart, RemoteUpload, CreateResult, CompleteResult } from './api/client.js';
+export { storageKey, loadSession, saveSession, clearSession, partSizeOf, memoryStore, } from './core/session.js';
+export type { SessionStatus, SessionRecord } from './core/session.js';
+export { classifyPut, classifyApi, backoffMs, RETRY_BUDGET } from './core/retry.js';
+export type { ErrorClass } from './core/retry.js';
+export { Emitter, computeProgress } from './core/events.js';
+export type { Progress } from './core/events.js';
+export { UploadEngine, IntegrityError, SYSTEMIC_TRANSPORT_FAILURE_THRESHOLD } from './core/engine.js';
+export type { CompleteInfo, EngineDeps, EngineOptions, EngineEvents, TransportDiagnostic } from './core/engine.js';
+export { buildUpload, findResumable } from './core/upload.js';
+export type { UploadOptions, Upload } from './core/upload.js';