@hostr/upload 0.1.0

package/README.md

@@ -0,0 +1,298 @@
+# @hostr/upload
+
+Resumable, presigned multipart uploads straight to R2 — bytes never transit
+the API Worker. Four entry points, each bundled independently:
+
+| Entry | Platform | Adapter-specific job |
+|---|---|---|
+| `@hostr/upload` | trusted backend only | `createUploadSession`, `mintUploadToken` — needs `HOSTR_API_KEY` |
+| `@hostr/upload/browser` | web | XHR transport (for upload-progress events), `localStorage` session store |
+| `@hostr/upload/node` | Node.js, scripts | `fetch` transport, filesystem session store |
+| `@hostr/upload/react-native` | Expo / RN | native staged-parts transport; every upload survives backgrounding |
+
+The engine, retry policy, and integrity checks are one shared core
+(`packages/upload/src/core`); adapters only supply platform I/O (read bytes,
+PUT bytes, persist a few KB of JSON). (The RN adapter is the exception: it
+bypasses the engine entirely for OS-carried native transfers — see the
+React Native section.)
+
+**One rule for clients of a proxying backend:** take the hostr `endpoint`
+from your backend's create-upload response (have your backend include its
+own `HOSTR_API_BASE`), never from separate client config. An upload token is
+only valid on the hostr instance that minted it, and endpoint drift fails as
+a bare mid-flow `401` with no hint of the cause — returning the endpoint
+with the session makes the drift structurally impossible. (Learned by the
+first consumer integration, 2026-08-22.)
+
+## Quickstart
+
+### Browser
+
+```js
+import { createUploadSession, createUpload } from '@hostr/upload/browser';
+
+// Trusted-backend call — holds HOSTR_API_KEY. Runs on your server, not in
+// the bundle you ship to users; shown inline here only for brevity.
+const session = await createUploadSession({
+  endpoint, apiKey, filename: file.name, contentType: file.type || 'video/mp4', size: file.size,
+});
+
+// From here on the browser holds only a scoped token — never the master key.
+const upload = await createUpload({ endpoint, uploadToken: session.uploadToken, assetId: session.assetId, file });
+
+upload.on('progress', p => {
+  console.log(`${p.partsDone}/${p.partsTotal} parts, ${(p.bytesConfirmed / p.totalBytes * 100).toFixed(1)}%`);
+  for (const [n, st] of Object.entries(p.partStates ?? {})) mark(Number(n), st); // 'inflight' | 'done', for a per-part grid
+});
+upload.on('statechange', s => console.log('state:', s)); // idle → running → (stalled ↔ running)* → completing → complete
+upload.start();
+
+const { etag, sha256 } = await upload.done;
+```
+
+Session state (assetId, part progress, hash midstate) persists under
+`localStorage` keyed by `name:size:lastModified`. Reload the page, pick the
+same file again, and `createUpload` resumes it automatically — no explicit
+"resume" call. See `apps/playground/upload.html` for the full flow, including
+re-minting a token for a saved session with `mintUploadToken`.
+
+### Node
+
+```js
+import { createUploadSession, createUpload } from '@hostr/upload/node';
+
+const session = await createUploadSession({
+  endpoint, apiKey, filename: 'video.mp4', contentType: 'video/mp4', size, partSize: PART_SIZE,
+});
+
+const upload = await createUpload({
+  endpoint, uploadToken: session.uploadToken, assetId: session.assetId,
+  file: '/path/to/video.mp4',      // filesystem path, not an open handle
+  stateDir: './upload-state',      // survives a process restart; defaults to os.tmpdir()/hostr-upload
+});
+
+upload.start();
+const { etag, sha256 } = await upload.done;
+```
+
+`infra/scripts/test-upload.mjs` is the canonical example — it drives the SDK
+through a kill/resume cycle and a corrupted-part self-heal against a real API
+and real R2.
+
+### React Native
+
+Requires `expo-file-system` >= 57 (Expo SDK 57): the adapter rides the modern
+File/Directory API — `FileHandle` byte-window reads, native per-part `md5`,
+`File.createUploadTask` background sessions — with no fallback to the removed
+`expo-file-system/legacy` surface. Older SDKs should pin an older version of
+this package.
+
+There is **one upload path on RN**. Parts are staged natively and carried by
+OS background-session tasks, so every upload keeps transferring if the app is
+backgrounded — that is not a mode, it's a property. The engine-style surface
+(`progress`/`statechange` events, `pause`/`resume`/`abort`, a `done` promise)
+wraps it. RN uploads record no whole-file sha256; the composite etag —
+verified per part and again at complete — is the integrity check.
+
+```tsx
+import { createUploadSession, mintUploadToken } from '@hostr/upload';
+import { createUpload, asyncStorageStore, reconcileOnLaunch } from '@hostr/upload/react-native';
+import AsyncStorage from '@react-native-async-storage/async-storage';
+
+const store = asyncStorageStore(AsyncStorage); // REQUIRED — RN has no default persisted store
+
+// Once, at app launch, before the user has picked anything — recovers any
+// upload a previous launch left mid-flight (including after a force-kill).
+useEffect(() => {
+  reconcileOnLaunch(store, { endpoint }).catch(() => {}); // best-effort; a real failure just waits for the next launch
+}, []);
+
+const session = await createUploadSession({ endpoint, apiKey, filename, contentType, size }); // trusted backend
+const refreshToken = assetId => mintUploadToken({ endpoint, apiKey, assetId }).then(r => r.uploadToken);
+
+const upload = await createUpload({
+  endpoint, uploadToken: session.uploadToken, assetId: session.assetId,
+  file,                    // { uri, size, modificationTime } from expo-document-picker / expo-image-picker
+  store,
+  onTokenExpired: refreshToken,
+});
+upload.on('progress', setProgress);
+upload.start();
+const { etag } = await upload.done; // resolved by the auto-reconcile that runs as the last part lands
+```
+
+For headless control (no events, no `done` promise — e.g. an upload queue
+worker), `startBackgroundUpload` returns the underlying handle directly:
+`reconcile()`, `pause()`, `abort()`, and session geometry. `reconcileOnLaunch`
+is the mandatory recovery hook either way.
+
+`apps/expo-playground/App.tsx` exercises the path end to end, with a manual
+"Reconcile now" button for the force-kill recovery leg.
+
+## Auth flow
+
+```
+trusted backend (holds HOSTR_API_KEY)
+  │  POST /v1/uploads  ──createUploadSession()──▶  { assetId, uploadToken, tokenExpiresAt, partSize, expectedParts }
+  ▼
+client (browser / Node / RN) — holds only { endpoint, uploadToken, assetId }, never the master key
+  │  createUpload({ endpoint, uploadToken, assetId, file, ... })
+  ▼
+GET /v1/uploads/:id · POST /v1/uploads/:id/parts · POST /v1/uploads/:id/complete
+  — every call authenticates with uploadToken; the API accepts either the
+    master key or an HMAC token scoped to exactly that assetId (401 on any
+    other asset — verified by infra/scripts/test-upload.mjs's leg 4)
+```
+
+The split matters because `createUploadSession`/`mintUploadToken` are the
+only two calls in this package that take `apiKey` — everything else takes
+`uploadToken`. Never ship `apiKey` to an untrusted client; it is the same
+credential that can create and delete any asset.
+
+**Token TTL** defaults to 72 h, clamped 5 min–7 d server-side
+(`clampTokenTtl` in `apps/api/src/limits.ts`) — pass `tokenTtlSec` to
+`createUploadSession` or `mintUploadToken` to override, within that range.
+
+**Token expiry mid-upload.** A large file can outlive a 72 h token. Pass
+`onTokenExpired: async (assetId) => newToken` to `createUpload` /
+`startBackgroundUpload`; the engine calls it on the first 401 from any API
+call, swaps in the returned token, and retries — no restart, no lost
+progress. The hook's server-side counterpart is `POST /v1/uploads/:id/token`
+(`mintUploadToken`), which is **master-key-only**: the untrusted client can
+never mint its own replacement, only ask the trusted backend to.
+
+## Integrity: three layers
+
+Each layer proves a different thing; none subsumes the others.
+
+1. **Per-part MD5 vs R2's ETag.** Checked immediately after every `PUT` — a
+   2xx response whose returned ETag doesn't match the client's MD5 of the
+   bytes it just sent is treated as `corrupt`, not success, and the part is
+   re-sent (see `infra/scripts/test-upload.mjs` leg 3, which flips a byte on
+   the wire and watches the engine self-heal). Proves: *these specific bytes,
+   for this one part, landed uncorrupted.*
+2. **Composite ETag at complete.** R2's multipart ETag is
+   `md5(concat(part MD5s)) + "-" + partCount` — computed client-side
+   (`compositeEtag`) from the same per-part MD5s and compared against what
+   `POST /complete` actually returns from R2's `CompleteMultipartUpload`.
+   Proves: *the parts assembled, in order, into one coherent object* — not
+   just that each part landed individually. This is also why multipart
+   ETags are **not** a portable content identity (CLAUDE.md's "Environment
+   gotchas"): the same file uploaded with a different part size produces a
+   different composite ETag.
+3. **Whole-file SHA-256.** Hashed incrementally as bytes are read — off the
+   part-size grid entirely — and included in `POST /complete`, stored as
+   asset metadata. Proves: *the identity of the file itself*, independent of
+   how it happened to be chunked or stored. This is the layer
+   `infra/scripts/verify-upload.mjs` uses to independently re-derive from
+   the bytes actually sitting in R2, with no input from the SDK at all.
+   Opt out with `checksum: 'none'` (skips the hash lane; use when content
+   identity isn't needed and the file is large enough that the extra read
+   pass matters).
+
+## Resume semantics
+
+**What survives a kill:** the session record (assetId, token, `partSize`,
+which parts are confirmed done, SHA-256 hash midstate) is persisted to the
+adapter's `store` on every part landing and token refresh. The hash lane's
+own checkpoints (every 8 MiB) only update that in-memory record — they don't
+trigger a save of their own; they ride along on the next part-completion
+save, or on a pause/stop (which flushes midstate explicitly so a kill mid-hash
+doesn't restart it from byte 0). Relaunching `createUpload` against
+the *same file* (same `name:size:lastModified` / `path:size:mtime` /
+`uri:size:modificationTime` — a modified file gets a fresh session instead
+of a corrupted resume) picks the session back up automatically.
+
+**What doesn't survive:** a part that was mid-flight at the moment of the
+kill is not resumed partway — it re-uploads from the start of that part.
+SHA-256 progress since the last checkpoint (every 8 MiB) is lost and
+re-hashed from the checkpoint forward, not from byte 0.
+
+**`GET /v1/uploads/:id`'s `parts[]` (R2's own `ListParts`) is the truth on
+every resume — never the persisted `done` map by itself.** The engine's
+`reconcile()` step, run before dispatching any work:
+
+- drops a locally-claimed-done part that R2 doesn't actually have (stale
+  claim — never sent, or R2 expired the multipart upload);
+- keeps a part whose stored MD5 matches R2's ETag for it (still verified);
+- re-uploads a part whose stored MD5 does **not** match R2's ETag (the
+  landed bytes aren't provably ours);
+- for a part R2 holds but local state has no record of at all — lost
+  storage, reinstall, a different device picking up the same file — **re-reads
+  and re-hashes that byte range locally**, and adopts it as done only if the
+  hash matches R2's ETag. Lost local state never gets a free pass; it has to
+  re-earn trust by re-hashing, exactly like a normal upload would.
+
+## Mobile
+
+### Android
+
+No second transport, and this package ships no native Android code. A JS
+engine keeps running for as long as something keeps the host process
+alive — on Android that means running the upload inside a **foreground
+service** with a visible notification (Android's own requirement for
+long-running background work; this is what YouTube/Drive do). Wire one up
+with `react-native-background-actions`-style patterns, or Expo's own
+foreground-service APIs, and call the normal foreground `createUpload` from
+inside it — the engine itself is unmodified; only what keeps the process
+alive changes.
+
+### iOS background caveats
+
+`startBackgroundUpload` hands each remaining part to
+`expo-file-system`'s iOS **background session** upload task, which the OS
+keeps carrying even if the app is backgrounded or its JS runtime is
+suspended — Expo ships the `AppDelegate` glue for this, so no native wiring
+is needed in an Expo app.
+
+- **A user force-killing the app cancels the OS's in-flight background
+  transfers outright.** This is documented `URLSessionUploadTask` behavior,
+  true for every app on iOS (YouTube included) — not a hostr limitation, and
+  not preventable from JS. `reconcileOnLaunch` on the next app start is the
+  recovery path, not a workaround: it re-derives from `ListParts` what
+  actually landed and re-stages the rest with a fresh presign.
+- **Scheduling is discretionary.** The OS decides when a background transfer
+  actually runs — battery state, network conditions, and Low Power Mode can
+  all delay it. There is no guaranteed latency.
+- **Completion callbacks are an optimization, never load-bearing.** A task's
+  completion handler, while the app happens to be alive to receive it, marks
+  a part done a little sooner and skips a round trip — but `reconcile()`
+  (via `ListParts`) is the only path anything is actually allowed to depend
+  on, since a callback after a kill is simply never delivered.
+- **No whole-file SHA-256.** `startBackgroundUpload` never runs the hash lane
+  (there's no in-process read pass over the whole file to drive it), so a
+  background-completed upload's asset record has no `sha256`; integrity
+  rests on the per-part MD5 and composite ETag checks alone (layers 1 and 2
+  above — see "Integrity: three layers").
+
+## Bundle identity: import error classes from the entry you use
+
+`NetworkError` and `ApiError` (and the engine's `IntegrityError`) are
+exported from every entry that re-exports `api/client.ts` — but **each
+subpath entry is bundled independently** (`@hostr/upload`, `/browser`,
+`/node`, `/react-native` are four separate build outputs). A class
+constructed inside one entry's bundle is not `instanceof`-equal to the "same"
+class imported from a different entry.
+
+This matters concretely if you supply a custom `transport` — e.g. to inject
+failures in a test, as `infra/scripts/test-upload.mjs` does — and want the
+engine's retry logic to recognize a thrown error as network-class:
+
+```js
+import { NetworkError } from '@hostr/upload/node'; // matches the entry createUpload was imported from
+// import { NetworkError } from '@hostr/upload';    // WRONG here — different bundle, fails the instanceof check
+```
+
+Always import `NetworkError`/`ApiError` from the same subpath you imported
+`createUpload` from.
+
+## See also
+
+- Root `README.md`, "Upload API" section — the HTTP surface this SDK talks
+  to (`POST /v1/uploads`, `GET /v1/uploads/:id`, `.../parts`,
+  `.../complete`, `.../token`, `DELETE`).
+- `pnpm build:sdk` — builds this package and vendors the browser bundle into
+  `apps/playground/vendor/` for the playground to import.
+- `pnpm verify:upload <assetId>` — independently re-derives the composite
+  ETag and SHA-256 from the bytes actually sitting in R2, with no input from
+  this SDK.

package/dist/api/client.d.ts

@@ -0,0 +1,71 @@
+/** Typed client for the hostr upload API (apps/api). No transport concerns
+ *  beyond fetch — retries, backoff, and offline handling live in the engine. */
+export declare class ApiError extends Error {
+    readonly status: number;
+    readonly code?: string;
+    constructor(status: number, code?: string);
+}
+export declare class NetworkError extends Error {
+    constructor(message?: string);
+}
+export type RemotePart = {
+    partNumber: number;
+    size: number;
+    etag: string;
+};
+export type RemoteUpload = {
+    assetId: string;
+    status: string;
+    key: string;
+    partSize: number;
+    expectedParts: number;
+    parts: RemotePart[];
+};
+export type CreateResult = {
+    assetId: string;
+    uploadToken: string;
+    tokenExpiresAt: number;
+    partSize: number;
+    expectedParts: number;
+    key: string;
+};
+export type CompleteResult = {
+    status: string;
+    etag?: string;
+    bytes?: number;
+};
+export declare class HostrApi {
+    #private;
+    constructor(endpoint: string, bearer: string, fetchFn?: typeof fetch);
+    setBearer(bearer: string): void;
+    getUpload(assetId: string): Promise<RemoteUpload>;
+    presignParts(assetId: string, partNumbers: number[], expiresIn?: number): Promise<{
+        urls: {
+            partNumber: number;
+            url: string;
+        }[];
+        expiresIn: number;
+    }>;
+    complete(assetId: string, body: {
+        sha256?: string;
+    }): Promise<CompleteResult>;
+    abort(assetId: string): Promise<void>;
+}
+export declare function createUploadSession(opts: {
+    endpoint: string;
+    apiKey: string;
+    filename: string;
+    contentType?: string;
+    size: number;
+    partSize?: number;
+    tokenTtlSec?: number;
+}, fetchFn?: typeof fetch): Promise<CreateResult>;
+export declare function mintUploadToken(opts: {
+    endpoint: string;
+    apiKey: string;
+    assetId: string;
+    tokenTtlSec?: number;
+}, fetchFn?: typeof fetch): Promise<{
+    uploadToken: string;
+    tokenExpiresAt: number;
+}>;

package/dist/browser/index.d.ts

@@ -0,0 +1,50 @@
+/** Browser adapter. This is a subpath entry (`@hostr/upload/browser`), never
+ *  re-exported from the package root — pulling DOM-only types (XHR, window,
+ *  localStorage) into the root entry would break non-browser bundlers. */
+import type { FileSource, HttpTransport, StateStore, NetworkMonitor } from '../core/types.js';
+import { type Upload } from '../core/upload.js';
+export { createUploadSession, mintUploadToken } from '../api/client.js';
+export { NetworkError, ApiError } from '../api/client.js';
+export { IntegrityError } from '../core/engine.js';
+export type { Upload } from '../core/upload.js';
+/** Structural File-ish source: anything with name/size/lastModified and a
+ *  slice(a, b).arrayBuffer() (a real DOM `File` satisfies this, and so does
+ *  a plain stub) is Node-testable without touching the DOM. Identity is
+ *  `name:size:lastModified` so a relaunch against the same file resolves to
+ *  the same session, and a changed file (different size or mtime) starts a
+ *  fresh one — mirrors nodeFileSource's `path:size:mtime`. */
+export declare function browserFileSource(f: {
+    name: string;
+    size: number;
+    lastModified: number;
+    slice(a: number, b: number): {
+        arrayBuffer(): Promise<ArrayBuffer>;
+    };
+}): FileSource;
+/** Default transport: XHR, not `fetch` — `fetch` exposes no upload-progress
+ *  events, and progress is the whole reason to reach for XHR here. */
+export declare const xhrTransport: HttpTransport;
+/** Persists session state under `localStorage` (or any Storage-ish backend,
+ *  for tests). Defaults to `globalThis.localStorage`, resolved lazily inside
+ *  the function body — not at module load — so importing this module in a
+ *  non-browser environment (e.g. Node tests) never throws; it only throws if
+ *  `localStorageStore()` is actually called with no `localStorage` global. */
+export declare function localStorageStore(ls?: Storage): StateStore;
+/** Wraps `navigator.onLine` + `window` `online`/`offline` events. Listeners
+ *  are attached lazily inside `onChange`, not at module top level or even at
+ *  factory-call time, so constructing (and even calling) this in Node —
+ *  where `window` doesn't exist — never throws; `onChange` just becomes a
+ *  no-op subscription in that case. */
+export declare function browserNetworkMonitor(): NetworkMonitor;
+export type CreateUploadOptions = {
+    endpoint: string;
+    uploadToken: string;
+    assetId: string;
+    file: File;
+    /** Defaults to a `localStorage`-backed store. */
+    store?: StateStore;
+    concurrency?: number;
+    checksum?: 'sha256' | 'none';
+    onTokenExpired?: (assetId: string) => Promise<string>;
+};
+export declare function createUpload(opts: CreateUploadOptions): Promise<Upload>;