@axium/storage 0.7.9 → 0.8.0

package/dist/server/api.js

@@ -0,0 +1,187 @@
+import { Permission } from '@axium/core';
+import { checkAuthForItem, checkAuthForUser } from '@axium/server/auth';
+import { config } from '@axium/server/config';
+import { database } from '@axium/server/database';
+import { error, parseBody, withError } from '@axium/server/requests';
+import { addRoute } from '@axium/server/routes';
+import { pick } from 'utilium';
+import * as z from 'zod';
+import { batchFormatVersion, StorageItemUpdate, syncProtocolVersion } from '../common.js';
+import '../polyfills.js';
+import { getLimits } from './config.js';
+import { currentUsage, deleteRecursive, getRecursive, parseItem } from './db.js';
+addRoute({
+    path: '/api/storage',
+    OPTIONS() {
+        return {
+            ...pick(config.storage, 'batch', 'chunk', 'max_chunks', 'max_transfer_size'),
+            syncProtocolVersion,
+            batchFormatVersion,
+        };
+    },
+});
+addRoute({
+    path: '/api/storage/item/:id',
+    params: { id: z.uuid() },
+    async GET(request, params) {
+        if (!config.storage.enabled)
+            error(503, 'User storage is disabled');
+        const itemId = params.id;
+        const { item } = await checkAuthForItem(request, 'storage', itemId, Permission.Read);
+        return parseItem(item);
+    },
+    async PATCH(request, params) {
+        if (!config.storage.enabled)
+            error(503, 'User storage is disabled');
+        const itemId = params.id;
+        const body = await parseBody(request, StorageItemUpdate);
+        await checkAuthForItem(request, 'storage', itemId, Permission.Manage);
+        const values = {};
+        if ('publicPermission' in body)
+            values.publicPermission = body.publicPermission;
+        if ('trash' in body)
+            values.trashedAt = body.trash ? new Date() : null;
+        if ('owner' in body)
+            values.userId = body.owner;
+        if ('name' in body)
+            values.name = body.name;
+        if (!Object.keys(values).length)
+            error(400, 'No valid fields to update');
+        return parseItem(await database
+            .updateTable('storage')
+            .where('id', '=', itemId)
+            .set(values)
+            .returningAll()
+            .executeTakeFirstOrThrow()
+            .catch(withError('Could not update item')));
+    },
+    async DELETE(request, params) {
+        if (!config.storage.enabled)
+            error(503, 'User storage is disabled');
+        const itemId = params.id;
+        const auth = await checkAuthForItem(request, 'storage', itemId, Permission.Manage);
+        const item = parseItem(auth.item);
+        await deleteRecursive(item.type != 'inode/directory', itemId);
+        return item;
+    },
+});
+addRoute({
+    path: '/api/storage/directory/:id',
+    params: { id: z.uuid() },
+    async GET(request, params) {
+        if (!config.storage.enabled)
+            error(503, 'User storage is disabled');
+        const itemId = params.id;
+        const { item } = await checkAuthForItem(request, 'storage', itemId, Permission.Read);
+        if (item.type != 'inode/directory')
+            error(409, 'Item is not a directory');
+        const items = await database
+            .selectFrom('storage')
+            .where('parentId', '=', itemId)
+            .where('trashedAt', 'is', null)
+            .selectAll()
+            .execute();
+        return items.map(parseItem);
+    },
+});
+addRoute({
+    path: '/api/storage/directory/:id/recursive',
+    params: { id: z.uuid() },
+    async GET(request, params) {
+        if (!config.storage.enabled)
+            error(503, 'User storage is disabled');
+        const itemId = params.id;
+        const { item } = await checkAuthForItem(request, 'storage', itemId, Permission.Read);
+        if (item.type != 'inode/directory')
+            error(409, 'Item is not a directory');
+        const items = await Array.fromAsync(getRecursive(itemId)).catch(withError('Could not get some directory items'));
+        return items;
+    },
+});
+addRoute({
+    path: '/api/users/:id/storage',
+    params: { id: z.uuid() },
+    async OPTIONS(request, params) {
+        if (!config.storage.enabled)
+            error(503, 'User storage is disabled');
+        const userId = params.id;
+        await checkAuthForUser(request, userId);
+        const [usage, limits] = await Promise.all([currentUsage(userId), getLimits(userId)]).catch(withError('Could not fetch data'));
+        return { usage, limits };
+    },
+    async GET(request, params) {
+        if (!config.storage.enabled)
+            error(503, 'User storage is disabled');
+        const userId = params.id;
+        await checkAuthForUser(request, userId);
+        const [items, usage, limits] = await Promise.all([
+            database.selectFrom('storage').where('userId', '=', userId).where('trashedAt', 'is', null).selectAll().execute(),
+            currentUsage(userId),
+            getLimits(userId),
+        ]).catch(withError('Could not fetch data'));
+        return { usage, limits, items: items.map(parseItem) };
+    },
+});
+addRoute({
+    path: '/api/users/:id/storage/root',
+    params: { id: z.uuid() },
+    async GET(request, params) {
+        if (!config.storage.enabled)
+            error(503, 'User storage is disabled');
+        const userId = params.id;
+        await checkAuthForUser(request, userId);
+        const items = await database
+            .selectFrom('storage')
+            .where('userId', '=', userId)
+            .where('trashedAt', 'is', null)
+            .where('parentId', 'is', null)
+            .selectAll()
+            .execute()
+            .catch(withError('Could not get storage items'));
+        return items.map(parseItem);
+    },
+});
+function existsInACL(column, userId) {
+    return (eb) => eb.exists(eb
+        .selectFrom('acl.storage')
+        .whereRef('itemId', '=', `item.${column}`)
+        .where('userId', '=', userId)
+        .where('permission', '!=', Permission.None));
+}
+addRoute({
+    path: '/api/users/:id/storage/shared',
+    params: { id: z.uuid() },
+    async GET(request, params) {
+        if (!config.storage.enabled)
+            error(503, 'User storage is disabled');
+        const userId = params.id;
+        await checkAuthForUser(request, userId);
+        const items = await database
+            .selectFrom('storage as item')
+            .selectAll('item')
+            .where('trashedAt', 'is', null)
+            .where(existsInACL('id', userId))
+            .where(eb => eb.not(existsInACL('parentId', userId)))
+            .execute()
+            .catch(withError('Could not get storage items'));
+        return items.map(parseItem);
+    },
+});
+addRoute({
+    path: '/api/users/:id/storage/trash',
+    params: { id: z.uuid() },
+    async GET(request, params) {
+        if (!config.storage.enabled)
+            error(503, 'User storage is disabled');
+        const userId = params.id;
+        await checkAuthForUser(request, userId);
+        const items = await database
+            .selectFrom('storage')
+            .where('userId', '=', userId)
+            .where('trashedAt', 'is not', null)
+            .selectAll()
+            .execute()
+            .catch(withError('Could not get trash'));
+        return items.map(parseItem);
+    },
+});

package/dist/server/batch.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/server/batch.js

@@ -0,0 +1,147 @@
+import { Permission } from '@axium/core';
+import * as acl from '@axium/server/acl';
+import { audit } from '@axium/server/audit';
+import { getSessionAndUser } from '@axium/server/auth';
+import config from '@axium/server/config';
+import { database } from '@axium/server/database';
+import { getToken, withError } from '@axium/server/requests';
+import { addRoute } from '@axium/server/routes';
+import { error } from '@sveltejs/kit';
+import { createHash } from 'node:crypto';
+import { unlinkSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path/posix';
+import * as z from 'zod';
+import { StorageBatchUpdate } from '../common.js';
+import { getLimits } from './config.js';
+import { currentUsage, getRecursiveIds, parseItem } from './db.js';
+addRoute({
+    path: '/api/storage/batch',
+    async POST(req) {
+        if (!config.storage.enabled)
+            error(503, 'User storage is disabled');
+        if (!config.storage.batch.enabled)
+            error(503, 'Batch updates are disabled');
+        const token = getToken(req);
+        if (!token)
+            error(401, 'Missing session token');
+        const { userId, user } = await getSessionAndUser(token).catch(withError('Invalid session token', 401));
+        if (user.isSuspended)
+            error(403, 'User is suspended');
+        const [usage, limits] = await Promise.all([currentUsage(userId), getLimits(userId)]).catch(withError('Could not fetch usage and/or limits'));
+        const batchHeaderSize = Number(req.headers.get('x-batch-header-size'));
+        if (!Number.isSafeInteger(batchHeaderSize) || batchHeaderSize < 2)
+            error(400, 'Invalid or missing header, X-Batch-Header-Size');
+        const size = Number(req.headers.get('content-length'));
+        if (Number.isNaN(size))
+            error(411, 'Missing or invalid content length header');
+        const raw = await req.bytes();
+        if (raw.byteLength - batchHeaderSize > size) {
+            await audit('storage_size_mismatch', userId, { item: null });
+            error(400, 'Content length does not match size header');
+        }
+        if (req.headers.get('content-type') != 'x-axium/storage-batch')
+            error(415, 'Invalid content type, expected x-axium/storage-batch');
+        let header;
+        try {
+            const text = new TextDecoder().decode(raw.subarray(0, batchHeaderSize));
+            header = StorageBatchUpdate.parse(JSON.parse(text));
+        }
+        catch (e) {
+            error(400, e instanceof z.core.$ZodError ? z.prettifyError(e) : 'invalid batch header');
+        }
+        const deletedIds = new Set(header.deleted);
+        const changedIds = new Set(Object.keys(header.content));
+        for (const id of [...Object.keys(header.metadata), ...changedIds]) {
+            if (deletedIds.has(id))
+                error(400, 'Item cannot be updated and deleted in the same batch: ' + id);
+        }
+        // checkAuthForItem but optimized to not re-fetch the session and also only run one DB query
+        const items = await database
+            .selectFrom('storage')
+            .selectAll()
+            .where('id', 'in', [...deletedIds, ...Object.keys(header.metadata), ...changedIds])
+            .select(acl.from('storage', { onlyId: userId }))
+            .$castTo()
+            .execute()
+            .catch(withError('Item(s) not found', 404));
+        for (const item of items) {
+            const permission = !changedIds.has(item.id) ? Permission.Manage : Permission.Edit;
+            if (changedIds.has(item.id)) {
+                // Extra checks for content changes
+                if (item.immutable)
+                    error(409, 'Item is immutable and cannot be modified: ' + item.id);
+                if (item.type == 'inode/directory')
+                    error(409, 'Directories do not have content');
+                if (item.trashedAt)
+                    error(410, 'Trashed items can not be changed');
+                if (limits.item_size && size > limits.item_size * 1_000_000)
+                    error(413, 'Item size exceeds maximum size: ' + item.id);
+            }
+            if (item.publicPermission >= permission)
+                continue;
+            if (userId == item.userId)
+                continue;
+            if (!item.acl || !item.acl.length)
+                error(403, 'Missing permission for item: ' + item.id);
+            const [control] = item.acl;
+            if (control.userId !== userId) {
+                await audit('acl_id_mismatch', userId, { item: item.id });
+                error(500, 'Access control entry does not match expected user ID');
+            }
+            if (control.permission >= permission)
+                continue;
+            error(403, 'Missing permission for item: ' + item.id);
+        }
+        if (limits.user_size && (usage.bytes + size - items.reduce((sum, item) => sum + item.size, 0)) / 1_000_000 >= limits.user_size)
+            error(413, 'Not enough space');
+        const tx = await database.startTransaction().execute();
+        const results = new Map();
+        try {
+            for (const [itemId, { offset, size }] of Object.entries(header.content)) {
+                const content = raw.subarray(offset, offset + size);
+                const hash = createHash('BLAKE2b512').update(content).digest();
+                const result = await tx
+                    .updateTable('storage')
+                    .where('id', '=', itemId)
+                    .set({ size, modifiedAt: new Date(), hash })
+                    .returningAll()
+                    .executeTakeFirstOrThrow();
+                writeFileSync(join(config.storage.data, result.id), content);
+                await tx.commit().execute();
+                results.set(itemId, parseItem(result));
+            }
+            const toDelete = await Array.fromAsync(getRecursiveIds(...header.deleted)).catch(withError('Could not get items to delete', 500));
+            const deleted = await tx.deleteFrom('storage').where('id', 'in', header.deleted).returningAll().execute();
+            for (const id of toDelete)
+                unlinkSync(join(config.storage.data, id));
+            for (const item of deleted)
+                results.set(item.id, parseItem(item));
+            for (const [itemId, update] of Object.entries(header.metadata)) {
+                const values = {};
+                if ('publicPermission' in update)
+                    values.publicPermission = update.publicPermission;
+                if ('trash' in update)
+                    values.trashedAt = update.trash ? new Date() : null;
+                if ('owner' in update)
+                    values.userId = update.owner;
+                if ('name' in update)
+                    values.name = update.name;
+                if (!Object.keys(values).length)
+                    error(400, 'No valid fields to update: ' + itemId);
+                const updated = await tx
+                    .updateTable('storage')
+                    .where('id', '=', itemId)
+                    .set(values)
+                    .returningAll()
+                    .executeTakeFirstOrThrow()
+                    .catch(withError('Could not update item: ' + itemId));
+                results.set(itemId, parseItem(updated));
+            }
+        }
+        catch (error) {
+            await tx.rollback().execute();
+            throw withError('Could not update item', 500)(error);
+        }
+        return Array.from(results.values());
+    },
+});

package/dist/server/cli.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/server/cli.js

@@ -0,0 +1,14 @@
+import { formatBytes } from '@axium/core/format';
+import { count, database } from '@axium/server/database';
+import { program } from 'commander';
+const cli = program.command('files').helpGroup('Plugins:').description('CLI integration for @axium/storage');
+cli.command('usage')
+    .description('Show storage usage information')
+    .action(async () => {
+    const { storage: items } = await count('storage');
+    const { size } = await database
+        .selectFrom('storage')
+        .select(eb => eb.fn.sum('size').as('size'))
+        .executeTakeFirstOrThrow();
+    console.log(`${items} items totaling ${formatBytes(Number(size))}`);
+});

package/dist/{server.d.ts → server/config.d.ts}

@@ -1,47 +1,10 @@
-import { type Schema } from '@axium/server/database';
-import type { Generated, Selectable } from 'kysely';
-import type { StorageItemMetadata, StorageLimits, StorageUsage } from './common.js';
-import './polyfills.js';
-declare module '@axium/server/database' {
-    interface Schema {
-        storage: {
-            createdAt: Generated<Date>;
-            hash: Uint8Array | null;
-            id: Generated<string>;
-            immutable: Generated<boolean>;
-            modifiedAt: Generated<Date>;
-            name: string;
-            parentId: string | null;
-            size: number;
-            trashedAt: Date | null;
-            type: string;
-            userId: string;
-            publicPermission: Generated<number>;
-            metadata: Generated<Record<string, unknown>>;
-        };
-    }
-    interface ExpectedSchema {
-        storage: ColumnTypes<Schema['storage']>;
-    }
-}
-/**
- * @internal A storage item selected from the database.
- */
-interface SelectedItem extends Selectable<Schema['storage']> {
-}
+import type { StorageLimits, StoragePublicConfig } from '../common.js';
+import '../polyfills.js';
 declare module '@axium/server/config' {
     interface Config {
-        storage: {
-            /** Whether the storage API endpoints are enabled */
-            enabled: boolean;
+        storage: StoragePublicConfig & {
             /** Whether the files app is enabled. Requires `enabled` */
             app_enabled: boolean;
-            /** Path to data directory */
-            data: string;
-            /** How many days files are kept in the trash */
-            trash_duration: number;
-            /** Default limits */
-            limits: StorageLimits;
             /** Content Addressable Storage (CAS) configuration */
             cas: {
                 /** Whether to use CAS */
@@ -51,9 +14,18 @@ declare module '@axium/server/config' {
                 /** Mime types to exclude when determining if CAS should be used */
                 exclude: string[];
             };
+            /** Path to data directory */
+            data: string;
+            /** Whether the storage API endpoints are enabled */
+            enabled: boolean;
+            /** Default limits */
+            limits: StorageLimits;
+            /** How many days files are kept in the trash */
+            trash_duration: number;
         };
     }
 }
+export declare const defaultCASMime: RegExp[];
 declare module '@axium/server/audit' {
     interface $EventTypes {
         storage_type_mismatch: {
@@ -67,21 +39,9 @@ declare module '@axium/server/audit' {
         };
     }
 }
-export interface StorageItem extends StorageItemMetadata {
-    data: Uint8Array<ArrayBufferLike>;
-}
-export declare function parseItem<T extends SelectedItem>(item: T): Omit<T, keyof Schema['storage']> & StorageItemMetadata;
-/**
- * Returns the current usage of the storage for a user in bytes.
- */
-export declare function currentUsage(userId: string): Promise<StorageUsage>;
-export declare function get(itemId: string): Promise<StorageItemMetadata>;
 export type ExternalLimitHandler = (userId?: string) => StorageLimits | Promise<StorageLimits>;
 /**
  * Define the handler to get limits for a user externally.
  */
 export declare function useLimits(handler: ExternalLimitHandler): void;
 export declare function getLimits(userId?: string): Promise<StorageLimits>;
-export declare function getRecursive(id: string): AsyncGenerator<string>;
-export declare function deleteRecursive(itemId: string, deleteSelf: boolean): Promise<void>;
-export {};

package/dist/server/config.js

@@ -0,0 +1,48 @@
+import { Severity } from '@axium/core';
+import { addEvent } from '@axium/server/audit';
+import { addConfigDefaults, config } from '@axium/server/config';
+import '../polyfills.js';
+export const defaultCASMime = [/video\/.*/, /audio\/.*/];
+addConfigDefaults({
+    storage: {
+        app_enabled: true,
+        batch: {
+            enabled: false,
+            max_items: 100,
+            max_item_size: 100,
+        },
+        cas: {
+            enabled: true,
+            include: [],
+            exclude: [],
+        },
+        chunk: false,
+        data: '/srv/axium/storage',
+        enabled: true,
+        limits: {
+            user_size: 1000,
+            item_size: 100,
+            user_items: 10_000,
+        },
+        max_chunks: 10,
+        max_transfer_size: 100,
+        trash_duration: 30,
+    },
+});
+addEvent({ source: '@axium/storage', name: 'storage_type_mismatch', severity: Severity.Warning, tags: ['mimetype'] });
+addEvent({ source: '@axium/storage', name: 'storage_size_mismatch', severity: Severity.Warning, tags: [] });
+let _getLimits = null;
+/**
+ * Define the handler to get limits for a user externally.
+ */
+export function useLimits(handler) {
+    _getLimits = handler;
+}
+export async function getLimits(userId) {
+    try {
+        return await _getLimits(userId);
+    }
+    catch {
+        return config.storage.limits;
+    }
+}

package/dist/server/db.d.ts

@@ -0,0 +1,47 @@
+import { type Schema } from '@axium/server/database';
+import type { Generated, Selectable } from 'kysely';
+import type { StorageItemMetadata, StorageUsage } from '../common.js';
+import '../polyfills.js';
+declare module '@axium/server/database' {
+    interface Schema {
+        storage: {
+            createdAt: Generated<Date>;
+            hash: Uint8Array | null;
+            id: Generated<string>;
+            immutable: Generated<boolean>;
+            modifiedAt: Generated<Date>;
+            name: string;
+            parentId: string | null;
+            size: number;
+            trashedAt: Date | null;
+            type: string;
+            userId: string;
+            publicPermission: Generated<number>;
+            metadata: Generated<Record<string, unknown>>;
+        };
+    }
+    interface ExpectedSchema {
+        storage: ColumnTypes<Schema['storage']>;
+    }
+}
+/**
+ * @internal A storage item selected from the database.
+ */
+export interface SelectedItem extends Selectable<Schema['storage']> {
+}
+export interface StorageItem extends StorageItemMetadata {
+    data: Uint8Array<ArrayBufferLike>;
+}
+export declare function parseItem<T extends SelectedItem>(item: T): Omit<T, keyof Schema['storage']> & StorageItemMetadata;
+/**
+ * Returns the current usage of the storage for a user in bytes.
+ */
+export declare function currentUsage(userId: string): Promise<StorageUsage>;
+export declare function get(itemId: string): Promise<StorageItemMetadata>;
+export declare function getRecursive(this: {
+    path: string;
+} | void, ...ids: string[]): AsyncGenerator<StorageItemMetadata & {
+    path: string;
+}>;
+export declare function getRecursiveIds(...ids: string[]): AsyncGenerator<string>;
+export declare function deleteRecursive(deleteSelf: boolean, ...itemId: string[]): Promise<void>;

package/dist/server/db.js

@@ -0,0 +1,77 @@
+import { config } from '@axium/server/config';
+import { database, expectedTypes } from '@axium/server/database';
+import { withError } from '@axium/server/requests';
+import { unlinkSync } from 'node:fs';
+import { join } from 'node:path/posix';
+import '../polyfills.js';
+expectedTypes.storage = {
+    createdAt: { type: 'timestamptz', required: true, hasDefault: true },
+    hash: { type: 'bytea' },
+    id: { type: 'uuid', required: true, hasDefault: true },
+    immutable: { type: 'bool', required: true },
+    modifiedAt: { type: 'timestamptz', required: true, hasDefault: true },
+    name: { type: 'text' },
+    parentId: { type: 'uuid' },
+    size: { type: 'int4', required: true },
+    trashedAt: { type: 'timestamptz' },
+    type: { type: 'text', required: true },
+    userId: { type: 'uuid', required: true },
+    publicPermission: { type: 'int4', required: true, hasDefault: true },
+    metadata: { type: 'jsonb', required: true, hasDefault: true },
+};
+export function parseItem(item) {
+    return {
+        ...item,
+        dataURL: `/raw/storage/${item.id}`,
+        hash: item.hash?.toHex?.(),
+    };
+}
+/**
+ * Returns the current usage of the storage for a user in bytes.
+ */
+export async function currentUsage(userId) {
+    const result = await database
+        .selectFrom('storage')
+        .where('userId', '=', userId)
+        .select(eb => eb.fn.countAll().as('items'))
+        .select(eb => eb.fn.sum('size').as('bytes'))
+        .executeTakeFirstOrThrow();
+    result.bytes = Number(result.bytes || 0);
+    result.items = Number(result.items);
+    return result;
+}
+export async function get(itemId) {
+    const result = await database
+        .selectFrom('storage')
+        .where('id', '=', itemId)
+        .selectAll()
+        .$narrowType()
+        .executeTakeFirstOrThrow();
+    return parseItem(result);
+}
+export async function* getRecursive(...ids) {
+    const items = await database.selectFrom('storage').where('parentId', 'in', ids).selectAll().execute();
+    for (const item of items) {
+        const path = this?.path ? this.path + '/' + item.name : item.name;
+        yield Object.assign(parseItem(item), { path });
+        if (item.type == 'inode/directory')
+            yield* getRecursive.call({ path }, item.id);
+    }
+}
+export async function* getRecursiveIds(...ids) {
+    const items = await database.selectFrom('storage').where('parentId', 'in', ids).select(['id', 'type']).execute();
+    for (const item of items) {
+        if (item.type != 'inode/directory')
+            yield item.id;
+        else
+            yield* getRecursiveIds(item.id);
+    }
+}
+export async function deleteRecursive(deleteSelf, ...itemId) {
+    const toDelete = await Array.fromAsync(getRecursiveIds(...itemId)).catch(withError('Could not get items to delete'));
+    if (deleteSelf)
+        toDelete.push(...itemId);
+    await database.deleteFrom('storage').where('id', '=', itemId).returningAll().execute().catch(withError('Could not delete item'));
+    for (const id of toDelete)
+        unlinkSync(join(config.storage.data, id));
+}

package/dist/server/hooks.d.ts

@@ -0,0 +1,9 @@
+import type { InitOptions, OpOptions } from '@axium/server/database';
+import '../common.js';
+import './index.js';
+export declare function statusText(): Promise<string>;
+export declare function init(): void;
+export declare function db_init(opt: InitOptions): Promise<void>;
+export declare function db_wipe(opt: OpOptions): Promise<void>;
+export declare function remove(opt: OpOptions): Promise<void>;
+export declare function clean(opt: OpOptions): Promise<void>;