@axium/storage 0.15.1 → 0.16.0

package/dist/client/api.d.ts

@@ -1,7 +1,23 @@
-import { StorageItemMetadata, type StorageItemUpdate, type UserStorage, type UserStorageInfo } from '../common.js';
+import type { StorageItemUpdate, UserStorage, UserStorageInfo } from '../common.js';
+import { StorageItemMetadata } from '../common.js';
+import '../polyfills.js';
 export interface UploadOptions {
     parentId?: string;
     name?: string;
+    onProgress?(this: void, uploaded: number, total: number): void;
+}
+declare global {
+    interface NetworkInformation {
+        readonly downlink: number;
+        readonly downlinkMax?: number;
+        readonly effectiveType: 'slow-2g' | '2g' | '3g' | '4g';
+        readonly rtt: number;
+        readonly saveData: boolean;
+        readonly type: 'bluetooth' | 'cellular' | 'ethernet' | 'none' | 'wifi' | 'wimax' | 'other' | 'unknown';
+    }
+    interface Navigator {
+        connection?: NetworkInformation;
+    }
 }
 export declare function uploadItem(file: Blob | File, opt?: UploadOptions): Promise<StorageItemMetadata>;
 export declare function updateItem(fileId: string, data: Blob): Promise<StorageItemMetadata>;

package/dist/client/api.js

@@ -1,13 +1,91 @@
 import { fetchAPI, prefix, token } from '@axium/client/requests';
-import { StorageItemMetadata } from '../common.js';
+import { blake2b } from 'blakejs';
 import { prettifyError } from 'zod';
-async function _upload(method, url, data, extraHeaders = {}) {
+import { StorageItemMetadata } from '../common.js';
+import '../polyfills.js';
+function rawStorage(suffix) {
+    const raw = '/raw/storage' + (suffix ? '/' + suffix : '');
+    if (prefix[0] == '/')
+        return raw;
+    const url = new URL(prefix);
+    url.pathname = raw;
+    return url;
+}
+const conTypeToSpeed = {
+    'slow-2g': 1,
+    '2g': 4,
+    '3g': 16,
+    '4g': 64,
+};
+async function handleError(response) {
+    if (response.headers.get('Content-Type')?.trim() != 'application/json')
+        throw await response.text();
+    const json = await response.json();
+    throw json.message;
+}
+export async function uploadItem(file, opt = {}) {
+    if (file instanceof File)
+        opt.name ||= file.name;
+    if (!opt.name)
+        throw 'item name is required';
+    const content = await file.bytes();
+    /**
+     * For big files, it takes a *really* long time to compute the hash, so we just don't do it ahead of time and leave it up to the server.
+     */
+    const hash = content.length < 10_000_000 ? blake2b(content).toHex() : null;
+    const upload = await fetchAPI('PUT', 'storage', {
+        parentId: opt.parentId,
+        name: opt.name,
+        type: file.type,
+        size: file.size,
+        hash,
+    });
+    if (upload.status == 'created')
+        return upload.item;
+    let chunkSize = upload.max_transfer_size * 1_000_000;
+    if (globalThis.navigator?.connection) {
+        chunkSize = Math.min(upload.max_transfer_size, conTypeToSpeed[globalThis.navigator.connection.effectiveType]) * 1_000_000;
+    }
+    let response;
+    for (let offset = 0; offset < content.length; offset += chunkSize) {
+        const size = Math.min(chunkSize, content.length - offset);
+        response = await fetch(rawStorage('chunk'), {
+            method: 'POST',
+            headers: {
+                'x-upload': upload.token,
+                'x-offset': offset.toString(),
+                'content-length': size.toString(),
+                'content-type': 'application/octet-stream',
+            },
+            body: content.slice(offset, offset + size),
+        });
+        if (!response.ok)
+            await handleError(response);
+        opt.onProgress?.(offset + size, content.length);
+        if (offset + size != content.length && response.status != 204)
+            console.warn('Unexpected end of upload before last chunk');
+    }
+    if (!response)
+        throw new Error('BUG: No response');
+    if (!response.headers.get('Content-Type')?.includes('application/json')) {
+        throw new Error(`Unexpected response type: ${response.headers.get('Content-Type')}`);
+    }
+    const json = await response.json().catch(() => ({ message: 'Unknown server error (invalid JSON response)' }));
+    if (!response.ok)
+        await handleError(response);
+    try {
+        return StorageItemMetadata.parse(json);
+    }
+    catch (e) {
+        throw prettifyError(e);
+    }
+}
+export async function updateItem(fileId, data) {
     const init = {
-        method,
+        method: 'POST',
         headers: {
             'Content-Type': data.type,
             'Content-Length': data.size.toString(),
-            ...extraHeaders,
         },
         body: data,
     };
@@ -15,7 +93,7 @@ async function _upload(method, url, data, extraHeaders = {}) {
         init.headers['X-Name'] = data.name;
     if (token)
         init.headers.Authorization = 'Bearer ' + token;
-    const response = await fetch(url, init);
+    const response = await fetch(rawStorage(fileId), init);
     if (!response.headers.get('Content-Type')?.includes('application/json')) {
         throw new Error(`Unexpected response type: ${response.headers.get('Content-Type')}`);
     }
@@ -29,25 +107,6 @@ async function _upload(method, url, data, extraHeaders = {}) {
         throw prettifyError(e);
     }
 }
-function rawStorage(fileId) {
-    const raw = '/raw/storage' + (fileId ? '/' + fileId : '');
-    if (prefix[0] == '/')
-        return raw;
-    const url = new URL(prefix);
-    url.pathname = raw;
-    return url;
-}
-export async function uploadItem(file, opt = {}) {
-    const headers = {};
-    if (opt.parentId)
-        headers['x-parent'] = opt.parentId;
-    if (opt.name)
-        headers['x-name'] = opt.name;
-    return await _upload('PUT', rawStorage(), file, headers);
-}
-export async function updateItem(fileId, data) {
-    return await _upload('POST', rawStorage(fileId), data);
-}
 export async function getItemMetadata(fileId) {
     return await fetchAPI('GET', 'storage/item/:id', undefined, fileId);
 }

package/dist/common.d.ts

@@ -160,9 +160,7 @@ export declare const StoragePublicConfig: z.ZodObject<{
         max_items: z.ZodInt;
         max_item_size: z.ZodInt;
     }, z.core.$strip>;
-    chunk: z.ZodBoolean;
     max_transfer_size: z.ZodInt;
-    max_chunks: z.ZodInt;
 }, z.core.$strip>;
 export interface StoragePublicConfig extends z.infer<typeof StoragePublicConfig> {
 }
@@ -172,9 +170,7 @@ export declare const StorageConfig: z.ZodObject<{
         max_items: z.ZodInt;
         max_item_size: z.ZodInt;
     }, z.core.$strip>;
-    chunk: z.ZodBoolean;
     max_transfer_size: z.ZodInt;
-    max_chunks: z.ZodInt;
     cas: z.ZodObject<{
         enabled: z.ZodOptional<z.ZodBoolean>;
         include: z.ZodOptional<z.ZodArray<z.ZodString>>;
@@ -188,12 +184,73 @@ export declare const StorageConfig: z.ZodObject<{
         user_size: z.ZodInt;
     }, z.core.$strip>;
     trash_duration: z.ZodNumber;
+    temp_dir: z.ZodString;
+    upload_timeout: z.ZodNumber;
 }, z.core.$strip>;
 declare module '@axium/core/plugins' {
     interface $PluginConfigs {
         '@axium/storage': z.infer<typeof StorageConfig>;
     }
 }
+export declare const StorageItemInit: z.ZodObject<{
+    name: z.ZodString;
+    size: z.ZodInt;
+    type: z.ZodString;
+    parentId: z.ZodOptional<z.ZodNullable<z.ZodUUID>>;
+    hash: z.ZodOptional<z.ZodNullable<z.ZodCustomStringFormat<"hex">>>;
+}, z.core.$strip>;
+export interface StorageItemInit extends z.infer<typeof StorageItemInit> {
+}
+export declare const UploadInitResult: z.ZodDiscriminatedUnion<[z.ZodObject<{
+    batch: z.ZodObject<{
+        enabled: z.ZodBoolean;
+        max_items: z.ZodInt;
+        max_item_size: z.ZodInt;
+    }, z.core.$strip>;
+    max_transfer_size: z.ZodInt;
+    status: z.ZodLiteral<"accepted">;
+    token: z.ZodBase64;
+}, z.core.$strip>, z.ZodObject<{
+    status: z.ZodLiteral<"created">;
+    item: z.ZodObject<{
+        createdAt: z.ZodCoercedDate<unknown>;
+        dataURL: z.ZodString;
+        hash: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+        id: z.ZodUUID;
+        immutable: z.ZodBoolean;
+        modifiedAt: z.ZodCoercedDate<unknown>;
+        name: z.ZodString;
+        userId: z.ZodUUID;
+        parentId: z.ZodNullable<z.ZodUUID>;
+        size: z.ZodInt;
+        trashedAt: z.ZodNullable<z.ZodCoercedDate<unknown>>;
+        type: z.ZodString;
+        metadata: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+        acl: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            itemId: z.ZodUUID;
+            userId: z.ZodOptional<z.ZodNullable<z.ZodUUID>>;
+            role: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+            tag: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+            user: z.ZodOptional<z.ZodNullable<z.ZodObject<{
+                id: z.ZodUUID;
+                name: z.ZodString;
+                email: z.ZodOptional<z.ZodEmail>;
+                emailVerified: z.ZodOptional<z.ZodOptional<z.ZodNullable<z.ZodCoercedDate<unknown>>>>;
+                image: z.ZodOptional<z.ZodNullable<z.ZodURL>>;
+                preferences: z.ZodOptional<z.ZodLazy<z.ZodObject<{
+                    debug: z.ZodDefault<z.ZodBoolean>;
+                }, z.core.$strip>>>;
+                roles: z.ZodArray<z.ZodString>;
+                tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
+                registeredAt: z.ZodCoercedDate<unknown>;
+                isAdmin: z.ZodOptional<z.ZodBoolean>;
+                isSuspended: z.ZodOptional<z.ZodBoolean>;
+            }, z.core.$strip>>>;
+            createdAt: z.ZodCoercedDate<unknown>;
+        }, z.core.$loose>>>;
+    }, z.core.$strip>;
+}, z.core.$strip>], "status">;
+export type UploadInitResult = z.infer<typeof UploadInitResult>;
 declare const StorageAPI: {
     readonly 'users/:id/storage': {
         readonly OPTIONS: z.ZodObject<{
@@ -380,12 +437,65 @@ declare const StorageAPI: {
                 max_items: z.ZodInt;
                 max_item_size: z.ZodInt;
             }, z.core.$strip>;
-            chunk: z.ZodBoolean;
             max_transfer_size: z.ZodInt;
-            max_chunks: z.ZodInt;
             syncProtocolVersion: z.ZodInt32;
             batchFormatVersion: z.ZodInt32;
         }, z.core.$strip>;
+        readonly PUT: readonly [z.ZodObject<{
+            name: z.ZodString;
+            size: z.ZodInt;
+            type: z.ZodString;
+            parentId: z.ZodOptional<z.ZodNullable<z.ZodUUID>>;
+            hash: z.ZodOptional<z.ZodNullable<z.ZodCustomStringFormat<"hex">>>;
+        }, z.core.$strip>, z.ZodDiscriminatedUnion<[z.ZodObject<{
+            batch: z.ZodObject<{
+                enabled: z.ZodBoolean;
+                max_items: z.ZodInt;
+                max_item_size: z.ZodInt;
+            }, z.core.$strip>;
+            max_transfer_size: z.ZodInt;
+            status: z.ZodLiteral<"accepted">;
+            token: z.ZodBase64;
+        }, z.core.$strip>, z.ZodObject<{
+            status: z.ZodLiteral<"created">;
+            item: z.ZodObject<{
+                createdAt: z.ZodCoercedDate<unknown>;
+                dataURL: z.ZodString;
+                hash: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+                id: z.ZodUUID;
+                immutable: z.ZodBoolean;
+                modifiedAt: z.ZodCoercedDate<unknown>;
+                name: z.ZodString;
+                userId: z.ZodUUID;
+                parentId: z.ZodNullable<z.ZodUUID>;
+                size: z.ZodInt;
+                trashedAt: z.ZodNullable<z.ZodCoercedDate<unknown>>;
+                type: z.ZodString;
+                metadata: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+                acl: z.ZodOptional<z.ZodArray<z.ZodObject<{
+                    itemId: z.ZodUUID;
+                    userId: z.ZodOptional<z.ZodNullable<z.ZodUUID>>;
+                    role: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+                    tag: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+                    user: z.ZodOptional<z.ZodNullable<z.ZodObject<{
+                        id: z.ZodUUID;
+                        name: z.ZodString;
+                        email: z.ZodOptional<z.ZodEmail>;
+                        emailVerified: z.ZodOptional<z.ZodOptional<z.ZodNullable<z.ZodCoercedDate<unknown>>>>;
+                        image: z.ZodOptional<z.ZodNullable<z.ZodURL>>;
+                        preferences: z.ZodOptional<z.ZodLazy<z.ZodObject<{
+                            debug: z.ZodDefault<z.ZodBoolean>;
+                        }, z.core.$strip>>>;
+                        roles: z.ZodArray<z.ZodString>;
+                        tags: z.ZodOptional<z.ZodArray<z.ZodString>>;
+                        registeredAt: z.ZodCoercedDate<unknown>;
+                        isAdmin: z.ZodOptional<z.ZodBoolean>;
+                        isSuspended: z.ZodOptional<z.ZodBoolean>;
+                    }, z.core.$strip>>>;
+                    createdAt: z.ZodCoercedDate<unknown>;
+                }, z.core.$loose>>>;
+            }, z.core.$strip>;
+        }, z.core.$strip>], "status">];
     };
     readonly 'storage/batch': {
         readonly POST: readonly [z.ZodArray<z.ZodObject<{

package/dist/common.js

@@ -79,12 +79,8 @@ export const StoragePublicConfig = z.object({
         /** Maximum size in KiB per item */
         max_item_size: z.int().positive(),
     }),
-    /** Whether to split files larger than `max_transfer_size` into multiple chunks */
-    chunk: z.boolean(),
     /** Maximum size in MiB per transfer/request */
     max_transfer_size: z.int().positive(),
-    /** Maximum number of chunks */
-    max_chunks: z.int().positive(),
 });
 export const StorageConfig = StoragePublicConfig.safeExtend({
     /** Content Addressable Storage (CAS) configuration */
@@ -106,8 +102,30 @@ export const StorageConfig = StoragePublicConfig.safeExtend({
     limits: StorageLimits,
     /** How many days files are kept in the trash */
     trash_duration: z.number(),
+    /** Where to put in-progress chunked uploads */
+    temp_dir: z.string(),
+    /** How many minutes before an in-progress upload times out */
+    upload_timeout: z.number(),
 });
 setServerConfig('@axium/storage', StorageConfig);
+export const StorageItemInit = z.object({
+    name: z.string(),
+    size: z.int().nonnegative(),
+    type: z.string(),
+    parentId: z.uuid().nullish(),
+    hash: z.hex().nullish(),
+});
+export const UploadInitResult = z.discriminatedUnion('status', [
+    StoragePublicConfig.safeExtend({
+        status: z.literal('accepted'),
+        /** Used for chunked uploads */
+        token: z.base64(),
+    }),
+    z.object({
+        status: z.literal('created'),
+        item: StorageItemMetadata,
+    }),
+]);
 const StorageAPI = {
     'users/:id/storage': {
         OPTIONS: UserStorageInfo,
@@ -127,6 +145,7 @@ const StorageAPI = {
             syncProtocolVersion: z.int32().nonnegative(),
             batchFormatVersion: z.int32().nonnegative(),
         }),
+        PUT: [StorageItemInit, UploadInitResult],
     },
     'storage/batch': {
         POST: [StorageBatchUpdate.array(), StorageItemMetadata.array()],

package/dist/polyfills.js

@@ -7,12 +7,17 @@ https://github.com/microsoft/TypeScript/issues/61695
 
 @todo Remove when TypeScript 5.9 is released
 */
-import { debug } from '@axium/core/node/io';
+import { debug } from '@axium/core/io';
 Uint8Array.prototype.toHex ??=
     (debug('Using a polyfill of Uint8Array.prototype.toHex'),
         function toHex() {
             return [...this].map(b => b.toString(16).padStart(2, '0')).join('');
         });
+Uint8Array.prototype.toBase64 ??=
+    (debug('Using a polyfill of Uint8Array.prototype.toBase64'),
+        function toBase64() {
+            return btoa(String.fromCharCode(...this));
+        });
 Uint8Array.fromHex ??=
     (debug('Using a polyfill of Uint8Array.fromHex'),
         function fromHex(hex) {

package/dist/server/api.js

@@ -1,24 +1,42 @@
 import { getConfig } from '@axium/core';
-import { checkAuthForItem, checkAuthForUser } from '@axium/server/auth';
+import { authRequestForItem, checkAuthForUser, requireSession } from '@axium/server/auth';
 import { database } from '@axium/server/database';
-import { error, parseBody, withError } from '@axium/server/requests';
+import { error, json, parseBody, withError } from '@axium/server/requests';
 import { addRoute } from '@axium/server/routes';
 import { pick } from 'utilium';
 import * as z from 'zod';
-import { batchFormatVersion, StorageItemUpdate, syncProtocolVersion } from '../common.js';
+import { batchFormatVersion, StorageItemInit, StorageItemUpdate, syncProtocolVersion } from '../common.js';
 import '../polyfills.js';
 import { getLimits } from './config.js';
 import { deleteRecursive, getRecursive, getUserStats, parseItem } from './db.js';
 import { from as aclFrom } from '@axium/server/acl';
+import { checkNewItem, createNewItem, startUpload } from './item.js';
 addRoute({
     path: '/api/storage',
     OPTIONS() {
         return {
-            ...pick(getConfig('@axium/storage'), 'batch', 'chunk', 'max_chunks', 'max_transfer_size'),
+            ...pick(getConfig('@axium/storage'), 'batch', 'max_transfer_size'),
             syncProtocolVersion,
             batchFormatVersion,
         };
     },
+    async PUT(request) {
+        const { enabled, ...rest } = getConfig('@axium/storage');
+        if (!enabled)
+            error(503, 'User storage is disabled');
+        const session = await requireSession(request);
+        const init = await parseBody(request, StorageItemInit);
+        const { existing } = await checkNewItem(init, session);
+        if (existing || init.type == 'inode/directory') {
+            const item = await createNewItem(init, session.userId);
+            return json({ status: 'created', item }, { status: 201 });
+        }
+        return json({
+            ...pick(rest, 'batch', 'max_transfer_size'),
+            status: 'accepted',
+            token: startUpload(init, session),
+        }, { status: 202 });
+    },
 });
 addRoute({
     path: '/api/storage/item/:id',
@@ -26,14 +44,14 @@ addRoute({
     async GET(request, { id: itemId }) {
         if (!getConfig('@axium/storage').enabled)
             error(503, 'User storage is disabled');
-        const { item } = await checkAuthForItem(request, 'storage', itemId, { read: true });
+        const { item } = await authRequestForItem(request, 'storage', itemId, { read: true });
         return parseItem(item);
     },
     async PATCH(request, { id: itemId }) {
         if (!getConfig('@axium/storage').enabled)
             error(503, 'User storage is disabled');
         const body = await parseBody(request, StorageItemUpdate);
-        await checkAuthForItem(request, 'storage', itemId, { manage: true });
+        await authRequestForItem(request, 'storage', itemId, { manage: true });
         const values = {};
         if ('trash' in body)
             values.trashedAt = body.trash ? new Date() : null;
@@ -54,7 +72,7 @@ addRoute({
     async DELETE(request, { id: itemId }) {
         if (!getConfig('@axium/storage').enabled)
             error(503, 'User storage is disabled');
-        const auth = await checkAuthForItem(request, 'storage', itemId, { manage: true });
+        const auth = await authRequestForItem(request, 'storage', itemId, { manage: true });
         const item = parseItem(auth.item);
         await deleteRecursive(item.type != 'inode/directory', itemId);
         return item;
@@ -66,7 +84,7 @@ addRoute({
     async GET(request, { id: itemId }) {
         if (!getConfig('@axium/storage').enabled)
             error(503, 'User storage is disabled');
-        const { item } = await checkAuthForItem(request, 'storage', itemId, { read: true });
+        const { item } = await authRequestForItem(request, 'storage', itemId, { read: true });
         if (item.type != 'inode/directory')
             error(409, 'Item is not a directory');
         const items = await database
@@ -85,7 +103,7 @@ addRoute({
     async GET(request, { id: itemId }) {
         if (!getConfig('@axium/storage').enabled)
             error(503, 'User storage is disabled');
-        const { item } = await checkAuthForItem(request, 'storage', itemId, { read: true });
+        const { item } = await authRequestForItem(request, 'storage', itemId, { read: true });
         if (item.type != 'inode/directory')
             error(409, 'Item is not a directory');
         const items = await Array.fromAsync(getRecursive(itemId)).catch(withError('Could not get some directory items'));

package/dist/server/item.d.ts

@@ -0,0 +1,26 @@
+import { type Session } from '@axium/core';
+import { type SessionAndUser } from '@axium/server/auth';
+import { type Hash } from 'node:crypto';
+import type { StorageItemInit, StorageItemMetadata } from '../common.js';
+import '../polyfills.js';
+export interface NewItemResult {
+    existing?: {
+        id: string;
+    };
+    needsHashing?: boolean;
+}
+export declare function useCAS(type: string): boolean;
+export declare function checkNewItem(init: StorageItemInit, session: SessionAndUser): Promise<NewItemResult>;
+export declare function createNewItem(init: StorageItemInit, userId: string, writeContent?: (path: string) => void): Promise<StorageItemMetadata>;
+export interface UploadInfo {
+    file: string;
+    fd: number;
+    hash: Hash;
+    uploadedBytes: number;
+    sessionId: string;
+    userId: string;
+    init: StorageItemInit;
+    remove(): void;
+}
+export declare function startUpload(init: StorageItemInit, session: Session): string;
+export declare function requireUpload(request: Request): Promise<UploadInfo>;

package/dist/server/item.js

@@ -0,0 +1,136 @@
+import { getConfig } from '@axium/core';
+import { authSessionForItem, requireSession } from '@axium/server/auth';
+import { database } from '@axium/server/database';
+import { error, withError } from '@axium/server/requests';
+import { createHash, randomBytes } from 'node:crypto';
+import { closeSync, linkSync, mkdirSync, openSync } from 'node:fs';
+import { join } from 'node:path';
+import * as z from 'zod';
+import '../polyfills.js';
+import { defaultCASMime, getLimits } from './config.js';
+import { getUserStats, parseItem } from './db.js';
+export function useCAS(type) {
+    const { cas } = getConfig('@axium/storage');
+    return !!(cas.enabled &&
+        type != 'inode/directory' &&
+        (defaultCASMime.some(pattern => pattern.test(type)) || cas.include?.some(mime => type.match(mime))));
+}
+export async function checkNewItem(init, session) {
+    const { userId } = session;
+    const { size, name, type, hash } = init;
+    const [usage, limits] = await Promise.all([getUserStats(userId), getLimits(userId)]).catch(withError('Could not fetch usage and/or limits'));
+    if (!name)
+        error(400, 'Missing name');
+    if (name.length > 255)
+        error(400, 'Name is too long');
+    const parentId = init.parentId
+        ? await z
+            .uuid()
+            .parseAsync(init.parentId)
+            .catch(() => error(400, 'Invalid parent ID'))
+        : null;
+    if (parentId)
+        await authSessionForItem('storage', parentId, { write: true }, session);
+    if (Number.isNaN(size))
+        error(411, 'Missing or invalid content length');
+    if (limits.user_items && usage.itemCount >= limits.user_items)
+        error(409, 'Too many items');
+    if (limits.user_size && (usage.usedBytes + size) / 1_000_000 >= limits.user_size)
+        error(413, 'Not enough space');
+    if (limits.item_size && size > limits.item_size * 1_000_000)
+        error(413, 'File size exceeds maximum size');
+    const isDirectory = type == 'inode/directory';
+    if (isDirectory && size > 0)
+        error(400, 'Directories can not have content');
+    if (!useCAS(type))
+        return {};
+    if (!hash)
+        return { needsHashing: true };
+    const existing = await database
+        .selectFrom('storage')
+        .select('id')
+        .where(eb => eb.and({ hash: Uint8Array.fromHex(hash), immutable: true }))
+        .limit(1)
+        .executeTakeFirst();
+    return { existing };
+}
+export async function createNewItem(init, userId, writeContent) {
+    const tx = await database.startTransaction().execute();
+    const { data: dataDir } = getConfig('@axium/storage');
+    const immutable = useCAS(init.type);
+    try {
+        const hash = typeof init.hash == 'string' ? Uint8Array.fromHex(init.hash) : null;
+        const existing = immutable
+            ? await database
+                .selectFrom('storage')
+                .select('id')
+                .where(eb => eb.and({ hash, immutable: true }))
+                .limit(1)
+                .executeTakeFirst()
+            : null;
+        const item = parseItem(await tx
+            .insertInto('storage')
+            .values({
+            ...init,
+            userId,
+            immutable,
+            hash,
+        })
+            .returningAll()
+            .executeTakeFirstOrThrow());
+        const path = join(dataDir, item.id);
+        if (existing)
+            linkSync(join(dataDir, existing.id), path);
+        else if (init.type != 'inode/directory') {
+            if (!writeContent)
+                error(501, 'Missing writeContent (this is a bug!)');
+            writeContent(path);
+        }
+        await tx.commit().execute();
+        return item;
+    }
+    catch (error) {
+        await tx.rollback().execute();
+        throw withError('Could not create item', 500)(error);
+    }
+}
+const inProgress = new Map();
+export function startUpload(init, session) {
+    const { temp_dir, upload_timeout } = getConfig('@axium/storage');
+    const token = randomBytes(32);
+    mkdirSync(temp_dir, { recursive: true });
+    const file = join(temp_dir, token.toHex());
+    const fd = openSync(file, 'a');
+    function remove() {
+        inProgress.delete(token.toBase64());
+        closeSync(fd);
+    }
+    inProgress.set(token.toBase64(), {
+        hash: createHash('BLAKE2b512'),
+        file,
+        fd,
+        uploadedBytes: 0,
+        sessionId: session.id,
+        userId: session.userId,
+        init,
+        remove,
+    });
+    setTimeout(() => {
+        remove();
+    }, upload_timeout * 60_000);
+    return token.toBase64();
+}
+export async function requireUpload(request) {
+    const token = request.headers.get('x-upload');
+    if (!token)
+        error(401, 'Missing upload token');
+    const upload = inProgress.get(token);
+    if (!upload)
+        error(400, 'Invalid upload token');
+    const session = await requireSession(request);
+    if (session.id != upload.sessionId)
+        error(403, 'Upload does not belong to the current session');
+    if (session.userId != upload.userId)
+        error(403, 'Upload does not belong to the current user');
+    return upload;
+}

package/dist/server/raw.js

@@ -52,95 +52,80 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
 });
 import { getConfig } from '@axium/core';
 import { audit } from '@axium/server/audit';
-import { checkAuthForItem, requireSession } from '@axium/server/auth';
+import { authRequestForItem, requireSession } from '@axium/server/auth';
 import { database } from '@axium/server/database';
 import { error, withError } from '@axium/server/requests';
 import { addRoute } from '@axium/server/routes';
 import { createHash } from 'node:crypto';
-import { closeSync, linkSync, openSync, readSync, writeFileSync } from 'node:fs';
+import { closeSync, openSync, readFileSync, readSync, renameSync, unlinkSync, writeFileSync, writeSync } from 'node:fs';
 import { join } from 'node:path/posix';
 import * as z from 'zod';
 import '../polyfills.js';
-import { defaultCASMime, getLimits } from './config.js';
+import { getLimits } from './config.js';
 import { getUserStats, parseItem } from './db.js';
+import { checkNewItem, createNewItem, requireUpload } from './item.js';
 addRoute({
     path: '/raw/storage',
     async PUT(request) {
         if (!getConfig('@axium/storage').enabled)
             error(503, 'User storage is disabled');
-        const { userId } = await requireSession(request);
-        const [usage, limits] = await Promise.all([getUserStats(userId), getLimits(userId)]).catch(withError('Could not fetch usage and/or limits'));
-        const name = request.headers.get('x-name');
-        if (!name)
-            error(400, 'Missing name header');
-        if (name.length > 255)
-            error(400, 'Name is too long');
-        const maybeParentId = request.headers.get('x-parent');
-        const parentId = maybeParentId
-            ? await z
-                .uuid()
-                .parseAsync(maybeParentId)
-                .catch(() => error(400, 'Invalid parent ID'))
-            : null;
-        if (parentId)
-            await checkAuthForItem(request, 'storage', parentId, { write: true });
-        const size = Number(request.headers.get('content-length'));
-        if (Number.isNaN(size))
-            error(411, 'Missing or invalid content length header');
-        if (limits.user_items && usage.itemCount >= limits.user_items)
-            error(409, 'Too many items');
-        if (limits.user_size && (usage.usedBytes + size) / 1_000_000 >= limits.user_size)
-            error(413, 'Not enough space');
-        if (limits.item_size && size > limits.item_size * 1_000_000)
-            error(413, 'File size exceeds maximum size');
+        const session = await requireSession(request);
+        const { userId } = session;
+        const name = request.headers.get('x-name'); // checked in `checkNewItem`
+        const parentId = request.headers.get('x-parent');
+        const size = Number(request.headers.get('x-size'));
+        const type = request.headers.get('content-type') || 'application/octet-stream';
         const content = await request.bytes();
         if (content.byteLength > size) {
             await audit('storage_size_mismatch', userId, { item: null });
             error(400, 'Content length does not match size header');
         }
-        const type = request.headers.get('content-type') || 'application/octet-stream';
-        const isDirectory = type == 'inode/directory';
-        if (isDirectory && size > 0)
-            error(400, 'Directories can not have content');
-        const useCAS = getConfig('@axium/storage').cas.enabled &&
-            !isDirectory &&
-            (defaultCASMime.some(pattern => pattern.test(type)) || getConfig('@axium/storage').cas.include?.some(mime => type.match(mime)));
-        const hash = isDirectory ? null : createHash('BLAKE2b512').update(content).digest();
-        const tx = await database.startTransaction().execute();
-        try {
-            const item = parseItem(await tx
-                .insertInto('storage')
-                .values({ userId, hash, name, size, type, immutable: useCAS, parentId })
-                .returningAll()
-                .executeTakeFirstOrThrow());
-            const path = join(getConfig('@axium/storage').data, item.id);
-            if (!useCAS) {
-                if (!isDirectory)
-                    writeFileSync(path, content);
-                await tx.commit().execute();
-                return item;
+        const hash = type == 'inode/directory' ? null : createHash('BLAKE2b512').update(content).digest();
+        const init = { name, size, type, parentId, hash: hash?.toHex() };
+        await checkNewItem(init, session);
+        return await createNewItem(init, userId, path => writeFileSync(path, content));
+    },
+});
+addRoute({
+    path: '/raw/storage/chunk',
+    async POST(request) {
+        if (!getConfig('@axium/storage').enabled)
+            error(503, 'User storage is disabled');
+        const upload = await requireUpload(request);
+        const size = Number(request.headers.get('content-length'));
+        if (Number.isNaN(size))
+            error(411, 'Missing or invalid content length');
+        if (upload.uploadedBytes + size > upload.init.size)
+            error(413, 'Upload exceeds allowed size');
+        const content = await request.bytes();
+        if (content.byteLength != size) {
+            await audit('storage_size_mismatch', upload.userId, { item: null });
+            error(400, `Content length mismatch: expected ${size}, got ${content.byteLength}`);
+        }
+        const offset = Number(request.headers.get('x-offset'));
+        if (offset != upload.uploadedBytes)
+            error(400, `Expected offset ${upload.uploadedBytes} but got ${offset}`);
+        writeSync(upload.fd, content); // opened with 'a', this appends
+        upload.hash.update(content);
+        upload.uploadedBytes += size;
+        if (upload.uploadedBytes != upload.init.size)
+            return new Response(null, { status: 204 });
+        const hash = upload.hash.digest();
+        upload.init.hash ??= hash.toHex();
+        if (hash.toHex() != upload.init.hash)
+            error(409, 'Hash mismatch');
+        upload.remove();
+        return await createNewItem(upload.init, upload.userId, path => {
+            try {
+                renameSync(upload.file, path);
             }
-            const existing = await tx
-                .selectFrom('storage')
-                .select('id')
-                .where('hash', '=', hash)
-                .where('id', '!=', item.id)
-                .limit(1)
-                .executeTakeFirst();
-            if (!existing) {
-                if (!isDirectory)
-                    writeFileSync(path, content);
-                await tx.commit().execute();
-                return item;
+            catch (e) {
+                if (e.code != 'EXDEV')
+                    throw e;
+                writeFileSync(path, readFileSync(upload.file));
+                unlinkSync(upload.file);
             }
-            linkSync(join(getConfig('@axium/storage').data, existing.id), path);
-            await tx.commit().execute();
-            return item;
-        }
-        catch (error) {
-            await tx.rollback().execute();
-            throw withError('Could not create item', 500)(error);
-        }
+        });
     },
 });
 addRoute({
@@ -151,7 +136,7 @@ addRoute({
         try {
             if (!getConfig('@axium/storage').enabled)
                 error(503, 'User storage is disabled');
-            const { item } = await checkAuthForItem(request, 'storage', itemId, { read: true });
+            const { item } = await authRequestForItem(request, 'storage', itemId, { read: true });
             if (item.trashedAt)
                 error(410, 'Trashed items can not be downloaded');
             const path = join(getConfig('@axium/storage').data, item.id);
@@ -198,7 +183,7 @@ addRoute({
     async POST(request, { id: itemId }) {
         if (!getConfig('@axium/storage').enabled)
             error(503, 'User storage is disabled');
-        const { item, session } = await checkAuthForItem(request, 'storage', itemId, { write: true });
+        const { item, session } = await authRequestForItem(request, 'storage', itemId, { write: true });
         if (item.immutable)
             error(405, 'Item is immutable');
         if (item.type == 'inode/directory')

package/lib/Add.svelte

@@ -7,7 +7,8 @@
 	const { parentId, onAdd }: { parentId?: string; onAdd?(item: StorageItemMetadata): void } = $props();
 
 	let uploadDialog = $state<HTMLDialogElement>()!;
-	let input = $state<HTMLInputElement>();
+	let uploadProgress = $state<[number, number][]>([]);
+	let input = $state<HTMLInputElement>()!;
 
 	let createDialog = $state<HTMLDialogElement>()!;
 	let createType = $state<string>();
@@ -43,13 +44,18 @@
 	bind:dialog={uploadDialog}
 	submitText="Upload"
 	submit={async () => {
-		for (const file of input?.files!) {
-			const item = await uploadItem(file, { parentId });
+		for (const [i, file] of Array.from(input.files!).entries()) {
+			const item = await uploadItem(file, {
+				parentId,
+				onProgress(uploaded, total) {
+					uploadProgress[i] = [uploaded, total];
+				},
+			});
 			onAdd?.(item);
 		}
 	}}
 >
-	<Upload bind:input multiple />
+	<Upload bind:input bind:progress={uploadProgress} multiple />
 </FormDialog>
 
 <FormDialog

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@axium/storage",
-	"version": "0.15.1",
+	"version": "0.16.0",
 	"author": "James Prevett <axium@jamespre.dev>",
 	"description": "User file storage for Axium",
 	"funding": {
@@ -41,11 +41,12 @@
 	"peerDependencies": {
 		"@axium/client": ">=0.13.0",
 		"@axium/core": ">=0.19.0",
-		"@axium/server": ">=0.34.0",
+		"@axium/server": ">=0.35.0",
 		"@sveltejs/kit": "^2.27.3",
 		"utilium": "^2.3.8"
 	},
 	"dependencies": {
+		"blakejs": "^1.2.1",
 		"zod": "^4.0.5"
 	},
 	"axium": {
@@ -80,7 +81,6 @@
 				"include": [],
 				"exclude": []
 			},
-			"chunk": false,
 			"data": "/srv/axium/storage",
 			"enabled": true,
 			"limits": {
@@ -88,9 +88,10 @@
 				"item_size": 100,
 				"user_items": 10000
 			},
-			"max_chunks": 10,
 			"max_transfer_size": 100,
-			"trash_duration": 30
+			"trash_duration": 30,
+			"temp_dir": "/tmp/axium",
+			"upload_timeout": 15
 		}
 	}
 }