@filelayer/core 0.3.0

package/dist/storage.d.ts

@@ -0,0 +1,269 @@
+/**
+ * STORAGE ADAPTERS
+ *
+ * P2 (no ambient authority) is why this interface is deliberately dumb: it
+ * takes an opaque key and moves bytes. It has no idea what an org is, cannot
+ * be asked "who may read this", and is never consulted during an authorization
+ * decision. If the adapter could answer access questions there would be two
+ * authorization paths, and the second one is always the one that leaks.
+ *
+ * Corollary: object keys are NOT secrets and are not treated as such anywhere.
+ *
+ * -----------------------------------------------------------------------------
+ * WHAT CHANGED, AND WHY
+ * -----------------------------------------------------------------------------
+ *
+ * 1. `provider` is now part of the interface.
+ *
+ *    `file.storage_provider` was written as the literal `'memory'` in
+ *    `Filelayer.upload()`, regardless of which adapter was configured. Against
+ *    `CREATE UNIQUE INDEX file_storage_key_idx ON file (storage_provider,
+ *    storage_key)` that means a production database records every object as
+ *    living in an in-memory store, and the one column that says WHERE the bytes
+ *    are is wrong for every row. An adapter must therefore be able to name
+ *    itself, and the name must come from the adapter rather than from the call
+ *    site.
+ *
+ * 2. Everything is streaming-capable.
+ *
+ *    `put()` took a `Uint8Array` and delivery returned one. That is a permanent
+ *    tax on the heap: a 2 GB upload was 2 GB of resident memory in the API
+ *    process, twice (once in the adapter, once in the response). `put()` now
+ *    accepts a `ReadableStream` and uses S3 multipart above a part threshold;
+ *    `stream()` returns a byte stream plus the metadata a correct HTTP response
+ *    needs, and supports ranged reads.
+ *
+ * 3. `head()` exists.
+ *
+ *    A `stat()` that has to download the object to learn its size is not a
+ *    stat.
+ *
+ * 4. `presignGet()` is OPTIONAL, and its optionality is the point.
+ *
+ *    It is the only capability the redirect delivery mode needs, and an adapter
+ *    that cannot mint a presigned URL simply does not offer redirect delivery
+ *    (`MemoryStorage` does not). Nothing else in the system may call it:
+ *    a presigned URL is authority that outlives the decision that produced it,
+ *    which is exactly the property P4 exists to deny, so it is reachable only
+ *    through the explicitly-acknowledged redirect mode in `delivery.ts`.
+ *
+ * 5. `list()` is OPTIONAL and exists for exactly one caller: orphan collection.
+ *
+ *    The storage write is not transactional (see `db.ts`). Bytes are written
+ *    before the metadata transaction commits, so a crash in between leaves an
+ *    object with no `file` row. That is a garbage-collection problem, not a
+ *    correctness one -- an orphan is unreachable, because every read path starts
+ *    from a `file` row -- but it is still our problem. `list()` is what makes it
+ *    collectable.
+ */
+/** Bytes, or a stream of bytes. Large uploads must use the second form. */
+export type PutBody = Uint8Array | ReadableStream<Uint8Array>;
+export interface StoragePutOptions {
+    /** Total length when known. Lets a small streaming put skip multipart. */
+    contentLength?: number;
+}
+export interface StoragePutResult {
+    /** Bytes actually written. Authoritative for `file.size_bytes`. */
+    bytes: number;
+    etag: string | null;
+}
+export interface ByteRange {
+    start: number;
+    /** Inclusive, per HTTP. Omit for "to the end". */
+    end?: number;
+}
+export interface ObjectHead {
+    size: number;
+    contentType: string | null;
+    etag: string | null;
+    lastModified: Date | null;
+}
+export interface ObjectStream {
+    body: ReadableStream<Uint8Array>;
+    /** Bytes in THIS response (the range length for a ranged read). */
+    size: number | null;
+    contentType: string | null;
+    etag: string | null;
+    /** Present only when the store honoured a range request. */
+    range?: {
+        start: number;
+        end: number;
+        total: number;
+    };
+}
+export interface ListEntry {
+    key: string;
+    size: number;
+    lastModified: Date | null;
+}
+export interface PresignOptions {
+    /** Seconds. The adapter clamps; the DELIVERY layer clamps harder. */
+    expiresInSeconds: number;
+    /** Forced onto the response by the object store, so the store cannot be
+     * tricked into serving our bytes with an attacker-chosen content type. */
+    responseContentType?: string;
+    responseContentDisposition?: string;
+}
+export interface StorageAdapter {
+    /**
+     * The value written to `file.storage_provider`. It participates in a UNIQUE
+     * index with the key, so it is part of an object's identity: change it for an
+     * existing deployment and every existing row points at the wrong store.
+     *
+     * Conventional values: 'memory', 's3', 'r2', 'gcs'.
+     */
+    readonly provider: string;
+    put(key: string, body: PutBody, contentType: string, opts?: StoragePutOptions): Promise<StoragePutResult>;
+    /** Buffered read. Only for callers that genuinely want the whole object. */
+    get(key: string): Promise<Uint8Array | null>;
+    stream(key: string, opts?: {
+        range?: ByteRange;
+    }): Promise<ObjectStream | null>;
+    head(key: string): Promise<ObjectHead | null>;
+    delete(key: string): Promise<void>;
+    /** Optional. Present => this adapter can support redirect delivery. */
+    presignGet?(key: string, opts: PresignOptions): Promise<string>;
+    /** Optional. Present => orphan collection can run against this adapter. */
+    list?(prefix: string, opts?: {
+        limit?: number;
+        cursor?: string | null;
+    }): Promise<{
+        entries: ListEntry[];
+        cursor: string | null;
+    }>;
+}
+/** Narrowing helpers, so callers do not hand-roll `typeof x.presignGet`. */
+export declare function canPresign(s: StorageAdapter): s is StorageAdapter & Required<Pick<StorageAdapter, 'presignGet'>>;
+export declare function canList(s: StorageAdapter): s is StorageAdapter & Required<Pick<StorageAdapter, 'list'>>;
+/** Collect a byte stream, refusing to exceed `limit`. */
+export declare function collectStream(stream: ReadableStream<Uint8Array>, limit?: number): Promise<Uint8Array>;
+export declare function bytesToStream(body: Uint8Array): ReadableStream<Uint8Array>;
+export declare class MemoryStorage implements StorageAdapter {
+    readonly provider = "memory";
+    private readonly objects;
+    put(key: string, body: PutBody, contentType: string): Promise<StoragePutResult>;
+    get(key: string): Promise<Uint8Array | null>;
+    head(key: string): Promise<ObjectHead | null>;
+    delete(key: string): Promise<void>;
+    stream(key: string, opts?: {
+        range?: ByteRange;
+    }): Promise<ObjectStream | null>;
+    list(prefix: string, opts?: {
+        limit?: number;
+        cursor?: string | null;
+    }): Promise<{
+        entries: ListEntry[];
+        cursor: string | null;
+    }>;
+    /**
+     * DELIBERATELY ABSENT: `presignGet`.
+     *
+     * There is no URL that reaches an in-process Map, so redirect delivery is
+     * structurally unavailable here rather than fake. A test that wants to
+     * exercise redirect delivery must run against something that can actually
+     * mint one -- which is the point.
+     */
+    /** Test-only: lets a test assert that delete really removed the bytes. */
+    keys(): string[];
+}
+export interface S3Config {
+    /** e.g. https://<account>.r2.cloudflarestorage.com, or a test server URL. */
+    endpoint: string;
+    bucket: string;
+    region: string;
+    accessKeyId: string;
+    secretAccessKey: string;
+    /** For STS / temporary credentials. Sent as `x-amz-security-token`. */
+    sessionToken?: string;
+    /**
+     * What lands in `file.storage_provider`. Defaults to 'r2' for an R2 endpoint
+     * and 's3' otherwise. It is part of an object's identity (see the UNIQUE
+     * index), so pin it explicitly for anything long-lived.
+     */
+    provider?: string;
+    /**
+     * `https://host/<bucket>/<key>` (default, and what R2 uses) vs
+     * `https://<bucket>.host/<key>`.
+     */
+    pathStyle?: boolean;
+    /** Multipart part size. S3 requires >= 5 MiB for all but the last part. */
+    partSizeBytes?: number;
+    /** Hard ceiling on a presigned URL's lifetime, in seconds. */
+    maxPresignSeconds?: number;
+}
+export declare class S3Storage implements StorageAdapter {
+    readonly provider: string;
+    private readonly cfg;
+    constructor(cfg: S3Config);
+    put(key: string, body: PutBody, contentType: string, opts?: StoragePutOptions): Promise<StoragePutResult>;
+    private putBuffer;
+    /**
+     * Streaming upload.
+     *
+     * The first `partSizeBytes` are buffered because we cannot know until we have
+     * them whether this is a one-shot PUT or a multipart upload, and a one-shot
+     * PUT needs `content-length` (and a payload hash) up front. Everything beyond
+     * that is uploaded part by part and never all resident at once, so peak
+     * memory is bounded by ONE part regardless of object size. That bound is the
+     * entire point of this method.
+     */
+    private putStream;
+    private createMultipartUpload;
+    private uploadPart;
+    private completeMultipartUpload;
+    private abortMultipartUpload;
+    get(key: string): Promise<Uint8Array | null>;
+    head(key: string): Promise<ObjectHead | null>;
+    stream(key: string, opts?: {
+        range?: ByteRange;
+    }): Promise<ObjectStream | null>;
+    delete(key: string): Promise<void>;
+    list(prefix: string, opts?: {
+        limit?: number;
+        cursor?: string | null;
+    }): Promise<{
+        entries: ListEntry[];
+        cursor: string | null;
+    }>;
+    /**
+     * A query-string-signed GET URL.
+     *
+     * READ THE WARNING IN `delivery.ts` BEFORE CALLING THIS. The URL is bearer
+     * authority that the object store will honour until it expires, and the
+     * object store has never heard of a grant, a revocation or an org. That is
+     * why it is not reachable from any ordinary delivery path.
+     *
+     * `response-content-type` and `response-content-disposition` are signed into
+     * the URL, so the object store -- not the client -- decides what the bytes are
+     * served as. Without them a redirect would drop the `nosniff`/`attachment`
+     * protections that `deliveryHeaders()` exists to guarantee.
+     */
+    presignGet(key: string, opts: PresignOptions): Promise<string>;
+    /**
+     * Path-style: `https://host/<bucket>/<key>`. Virtual-hosted:
+     * `https://<bucket>.host/<key>`.
+     *
+     * The canonical URI is built by RFC-3986-encoding each SEGMENT and is carried
+     * separately from `URL.pathname`, because `new URL()` normalises `.`/`..`
+     * segments and re-encodes some characters. Signing one string and sending
+     * another is the classic SigV4 bug and produces a 403 that looks like a
+     * credentials problem.
+     */
+    private objectUrl;
+    private sign;
+    private signedFetch;
+}
+/**
+ * RFC 3986 unreserved-set encoding.
+ *
+ * `encodeURIComponent` leaves `!'()*` alone and `encodeURI` additionally leaves
+ * `#?&=+,:;@$` alone. The previous implementation used `encodeURI` on the whole
+ * key, which meant a key containing `#` truncated the URL at the fragment, a key
+ * containing `?` started a query string, and a key containing `+` signed one
+ * byte sequence and sent another. Keys are constructed by us today, but "the
+ * caller never puts a `#` in a key" is not a property the type system carries.
+ */
+export declare function rfc3986(s: string): string;
+/** Sorted by encoded key, then encoded value. Empty values keep their `=`. */
+export declare function canonicalQueryString(query: Record<string, string>): string;
+//# sourceMappingURL=storage.d.ts.map

package/dist/storage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage.d.ts","sourceRoot":"","sources":["../src/storage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyDG;AAQH,2EAA2E;AAC3E,MAAM,MAAM,OAAO,GAAG,UAAU,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;AAE9D,MAAM,WAAW,iBAAiB;IAChC,0EAA0E;IAC1E,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,gBAAgB;IAC/B,mEAAmE;IACnE,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CACrB;AAED,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,kDAAkD;IAClD,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,YAAY,EAAE,IAAI,GAAG,IAAI,CAAC;CAC3B;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;IACjC,mEAAmE;IACnE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,4DAA4D;IAC5D,KAAK,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;CACvD;AAED,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,IAAI,GAAG,IAAI,CAAC;CAC3B;AAED,MAAM,WAAW,cAAc;IAC7B,qEAAqE;IACrE,gBAAgB,EAAE,MAAM,CAAC;IACzB;8EAC0E;IAC1E,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,0BAA0B,CAAC,EAAE,MAAM,CAAC;CACrC;AAED,MAAM,WAAW,cAAc;IAC7B;;;;;;OAMG;IACH,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE1B,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC1G,4EAA4E;IAC5E,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;IAC7C,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,SAAS,CAAA;KAAE,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;IAChF,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;IAC9C,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEnC,uEAAuE;IACvE,UAAU,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAChE,2EAA2E;IAC3E,IAAI,CAAC,CACH,MAAM,EAAE,MAAM,EACd,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,GAChD,OAAO,CAAC;QAAE,OAAO,EAAE,SAAS,EAAE,CAAC;QAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC,CAAC;CAC7D;AAED,4EAA4E;AAC5E,wBAAgB,UAAU,CACxB,CAAC,EAAE,cAAc,GAChB,CAAC,IAAI,cAAc,GAAG,QAAQ,CAAC,IAAI,CAAC,cAAc,EAAE,YAAY,CAAC,CAAC,CAEpE;AAED,wBAAgB,OAAO,CACrB,CAAC,EAAE,cAAc,GAChB,CAAC,IAAI,cAAc,GAAG,QAAQ,CAAC,IAAI,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC,CAE9D;AAMD,yDAAyD;AACzD,wBAAsB,aAAa,CACjC,MAAM,EAAE,cAAc,CAAC,UAAU,CAAC,EAClC,KAAK,SAAoB,GACxB,OAAO,CAAC,UAAU,CAAC,CA0BrB;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,UAAU,GAAG,cAAc,CAAC,UAAU,CAAC,CAO1E;AAMD,qBAAa,aAAc,YAAW,cAAc;IAClD,QAAQ,CAAC,QAAQ,YAAY;IAE7B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAGpB;IAEE,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAO/E,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAI5C,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAW7C,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIlC,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,GAAE;QAAE,KAAK,CAAC,EAAE,SAAS,CAAA;KAAO,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IAoBnF,IAAI,CACR,MAAM,EAAE,MAAM,EACd,IAAI,GAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;KAAO,GACpD,OAAO,CAAC;QAAE,OAAO,EAAE,SAAS,EAAE,CAAC;QAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC;IAiB3D;;;;;;;OAOG;IAEH,0EAA0E;IAC1E,IAAI,IAAI,MAAM,EAAE;CAGjB;AAkBD,MAAM,WAAW,QAAQ;IACvB,6EAA6E;IAC7E,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,MAAM,CAAC;IACxB,uEAAuE;IACvE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,2EAA2E;IAC3E,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,8DAA8D;IAC9D,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAKD,qBAAa,SAAU,YAAW,cAAc;IAC9C,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE1B,OAAO,CAAC,QAAQ,CAAC,GAAG,CAEU;gBAElB,GAAG,EAAE,QAAQ;IA2BnB,GAAG,CACP,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,OAAO,EACb,WAAW,EAAE,MAAM,EACnB,IAAI,GAAE,iBAAsB,GAC3B,OAAO,CAAC,gBAAgB,CAAC;YAKd,SAAS;IAgBvB;;;;;;;;;OASG;YACW,SAAS;YAwDT,qBAAqB;YAerB,UAAU;YAmBV,uBAAuB;YAiCvB,oBAAoB;IAO5B,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAU5C,IAAI,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAiB7C,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,GAAE;QAAE,KAAK,CAAC,EAAE,SAAS,CAAA;KAAO,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC;IAmCnF,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQlC,IAAI,CACR,MAAM,EAAE,MAAM,EACd,IAAI,GAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;KAAO,GACpD,OAAO,CAAC;QAAE,OAAO,EAAE,SAAS,EAAE,CAAC;QAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC;IA2B3D;;;;;;;;;;;;OAYG;IACG,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,MAAM,CAAC;IAoCpE;;;;;;;;;OASG;IACH,OAAO,CAAC,SAAS;IAejB,OAAO,CAAC,IAAI;YAcE,WAAW;CA+D1B;AAcD;;;;;;;;;GASG;AACH,wBAAgB,OAAO,CAAC,CAAC,EAAE,MAAM,GAAG,MAAM,CAKzC;AAED,8EAA8E;AAC9E,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,CAM1E"}