@promptbook/cli 0.103.0-46 → 0.103.0-47

package/apps/agents-server/src/app/api/upload/route.ts

@@ -0,0 +1,75 @@
+import { nextRequestToNodeRequest } from '@/src/utils/cdn/utils/nextRequestToNodeRequest';
+import { TODO_any } from '@promptbook-local/types';
+import { serializeError } from '@promptbook-local/utils';
+import formidable from 'formidable';
+import { readFile } from 'fs/promises';
+import { NextRequest, NextResponse } from 'next/server';
+import { assertsError } from '../../../../../../src/errors/assertsError';
+import { string_url } from '../../../../../../src/types/typeAliases';
+import { keepUnused } from '../../../../../../src/utils/organization/keepUnused';
+import { $provideCdnForServer } from '../../../../src/tools/$provideCdnForServer';
+import { getUserFileCdnKey } from '../../../../src/utils/cdn/utils/getUserFileCdnKey';
+import { validateMimeType } from '../../../../src/utils/validators/validateMimeType';
+
+export async function POST(request: NextRequest) {
+    try {
+        const nodeRequest = await nextRequestToNodeRequest(request);
+
+        const files = await new Promise<formidable.Files>((resolve, reject) => {
+            const form = formidable({});
+            form.parse(nodeRequest as TODO_any, (error, fields, files) => {
+                keepUnused(fields);
+
+                if (error) {
+                    return reject(error);
+                }
+                resolve(files);
+            });
+        });
+
+        const uploadedFiles = files.file;
+
+        if (!uploadedFiles || uploadedFiles.length !== 1) {
+            return NextResponse.json(
+                { message: 'In form data there is not EXACTLY one "file" field' },
+                { status: 400 },
+            );
+        }
+
+        const uploadedFile = uploadedFiles[0]!;
+        const fileBuffer = await readFile(uploadedFile.filepath);
+        const cdn = $provideCdnForServer();
+        const key = getUserFileCdnKey(fileBuffer, uploadedFile.originalFilename || uploadedFile.newFilename);
+
+        await cdn.setItem(key, {
+            type: validateMimeType(uploadedFile.mimetype),
+            data: fileBuffer,
+        });
+
+        const fileUrl = cdn.getItemUrl(key);
+
+        return NextResponse.json({ fileUrl: fileUrl.href as string_url }, { status: 201 });
+    } catch (error) {
+        assertsError(error);
+
+        console.error(error);
+
+        return new Response(
+            JSON.stringify(
+                serializeError(error),
+                // <- TODO: !!! Rename `serializeError` to `errorToJson`
+                null,
+                4,
+                // <- TODO: !!! Allow to configure pretty print for agent server
+            ),
+            {
+                status: 400, // <- TODO: !!! Make `errorToHttpStatusCode`
+                headers: { 'Content-Type': 'application/json' },
+            },
+        );
+    }
+}
+
+/**
+ * TODO: !!!! Is this Working on Vercel
+ */

package/apps/agents-server/src/tools/$provideAgentCollectionForServer.ts

@@ -6,6 +6,7 @@ import { getSupabaseForServer } from '../supabase/getSupabaseForServer';
 
 /**
  * Cache of provided agent collection
+ * 
  * @private internal cache for `$provideAgentCollectionForServer`
  */
 let agentCollection: null | AgentCollection = null;

package/apps/agents-server/src/tools/$provideCdnForServer.ts

@@ -0,0 +1,28 @@
+import { DigitalOceanSpaces } from '../utils/cdn/classes/DigitalOceanSpaces';
+import { IIFilesStorageWithCdn } from '../utils/cdn/interfaces/IFilesStorage';
+
+/**
+ * Cache of CDN instance
+ *
+ * @private internal cache for `$provideCdnForServer`
+ */
+let cdn: IIFilesStorageWithCdn | null = null;
+
+/**
+ * !!!
+ */
+export function $provideCdnForServer(): IIFilesStorageWithCdn {
+    if (!cdn) {
+        cdn = new DigitalOceanSpaces({
+            bucket: process.env.CDN_BUCKET!,
+            pathPrefix: process.env.NEXT_PUBLIC_CDN_PATH_PREFIX!,
+            endpoint: process.env.CDN_ENDPOINT!,
+            accessKeyId: process.env.CDN_ACCESS_KEY_ID!,
+            secretAccessKey: process.env.CDN_SECRET_ACCESS_KEY!,
+            cdnPublicUrl: new URL(process.env.NEXT_PUBLIC_CDN_PUBLIC_URL!),
+            gzip: true,
+        });
+    }
+
+    return cdn;
+}

package/apps/agents-server/src/utils/cdn/classes/DigitalOceanSpaces.ts

@@ -0,0 +1,119 @@
+import { GetObjectCommand, PutObjectCommand, PutObjectCommandInput, S3Client } from '@aws-sdk/client-s3';
+import { NotYetImplementedError } from '@promptbook-local/core';
+import { gzip, ungzip } from 'node-gzip';
+import { TODO_USE } from '../../../../../../src/utils/organization/TODO_USE';
+import { validateMimeType } from '../../validators/validateMimeType';
+import type { IFile, IIFilesStorageWithCdn } from '../interfaces/IFilesStorage';
+
+type IDigitalOceanSpacesConfig = {
+    readonly bucket: string;
+    readonly pathPrefix: string;
+    readonly endpoint: string;
+    readonly accessKeyId: string;
+    readonly secretAccessKey: string;
+    readonly cdnPublicUrl: URL;
+    readonly gzip: boolean;
+
+    // TODO: [⛳️] Probbably prefix should be in this config not on the consumer side
+};
+
+export class DigitalOceanSpaces implements IIFilesStorageWithCdn {
+    public get cdnPublicUrl() {
+        return this.config.cdnPublicUrl;
+    }
+
+    private s3: S3Client;
+
+    public constructor(private readonly config: IDigitalOceanSpacesConfig) {
+        this.s3 = new S3Client({
+            region: 'auto',
+            endpoint: 'https://' + config.endpoint,
+            credentials: {
+                accessKeyId: config.accessKeyId,
+                secretAccessKey: config.secretAccessKey,
+            },
+        });
+    }
+
+    public getItemUrl(key: string): URL {
+        return new URL(this.config.pathPrefix + '/' + key, this.cdnPublicUrl);
+    }
+
+    public async getItem(key: string): Promise<IFile | null> {
+        const parameters = {
+            Bucket: this.config.bucket,
+            Key: this.config.pathPrefix + '/' + key,
+        };
+
+        try {
+            const { Body, ContentType, ContentEncoding } = await this.s3.send(new GetObjectCommand(parameters));
+
+            // const blob = new Blob([await Body?.transformToByteArray()!]);
+
+            if (ContentEncoding === 'gzip') {
+                return {
+                    type: validateMimeType(ContentType),
+                    data: await ungzip(await Body!.transformToByteArray()),
+                };
+            } else {
+                return {
+                    type: validateMimeType(ContentType),
+                    data: (await Body!.transformToByteArray()) as Buffer,
+                };
+            }
+        } catch (error) {
+            if (error instanceof Error && error.name.match(/^NoSuchKey/)) {
+                return null;
+            } else {
+                throw error;
+            }
+        }
+    }
+
+    public async removeItem(key: string): Promise<void> {
+        TODO_USE(key);
+        throw new NotYetImplementedError(`DigitalOceanSpaces.removeItem is not implemented yet`);
+    }
+
+    public async setItem(key: string, file: IFile): Promise<void> {
+        // TODO: Put putObjectRequestAdditional into processedFile
+        const putObjectRequestAdditional: Partial<PutObjectCommandInput> = {};
+
+        let processedFile: IFile;
+        if (this.config.gzip) {
+            const gzipped = await gzip(file.data);
+            const sizePercentageAfterCompression = gzipped.byteLength / file.data.byteLength;
+            if (sizePercentageAfterCompression < 0.7) {
+                // consolex.log(`Gzipping ${key} (${Math.floor(sizePercentageAfterCompression * 100)}%)`);
+                processedFile = { ...file, data: gzipped };
+                putObjectRequestAdditional.ContentEncoding = 'gzip';
+            } else {
+                processedFile = file;
+                // consolex.log(`NOT Gzipping ${key} (${Math.floor(sizePercentageAfterCompression * 100)}%)`);
+            }
+        } else {
+            processedFile = file;
+        }
+
+        const uploadResult = await this.s3.send(
+            new PutObjectCommand({
+                Bucket: this.config.bucket,
+                Key: this.config.pathPrefix + '/' + key,
+                ContentType: processedFile.type,
+                ...putObjectRequestAdditional,
+                Body: processedFile.data,
+                // TODO: Public read access / just private to extending class
+                ACL: 'public-read',
+            }),
+        );
+
+        if (!uploadResult.ETag) {
+            throw new Error(`Upload result does not contain ETag`);
+        }
+    }
+}
+
+/**
+ * TODO: Implement Read-only mode
+ * TODO: [☹️] Unite with `PromptbookStorage` and move to `/src/...`
+ */

package/apps/agents-server/src/utils/cdn/interfaces/IFilesStorage.ts

@@ -0,0 +1,32 @@
+import type { string_mime_type } from '../../../../../../src/types/typeAliases';
+import type { IStorage } from './IStorage';
+
+export type IFile = {
+    // Maybe TODO name: string_name;
+    type: string_mime_type;
+    data: Buffer;
+};
+
+/**
+ * Represents storage that will store each keypair in a separate file.
+ */
+export type IFilesStorage = Omit<IStorage<IFile>, 'length' | 'clear' | 'key'>;
+
+/**
+ * Represents storage that can give public deterministic  URL for each file
+ */
+export type IIFilesStorageWithCdn = IFilesStorage & {
+    readonly cdnPublicUrl: URL;
+    getItemUrl(key: string): URL;
+};
+
+/**
+ * TODO: Probably not deterministic and async getItemUrl
+ * TODO: Probably just createUrlMaker
+ * TODO: List method
+ * TODO: Glob method
+ * TODO: Subfolder (similar to PrefixStorage) method
+ * TODO: Subscribe, list, sub(folder) should be part of LIB everstorage
+ * TODO: Probably implement observe through RxJS
+ * TODO: [☹️] Unite with `PromptbookStorage` and move to `/src/...`
+ */

package/apps/agents-server/src/utils/cdn/interfaces/IStorage.ts

@@ -0,0 +1,14 @@
+// Note: This is a simplified version of the IStorage interface based on the usage in the project.
+export type IStorage<T> = {
+    readonly length: Promise<number>;
+    clear(): Promise<void>;
+    getItem(key: string): Promise<T | null>;
+    key(index: number): Promise<string | null>;
+    removeItem(key: string): Promise<void>;
+    setItem(key: string, value: T): Promise<void>;
+};
+
+
+/**
+ * TODO: [☹️] Unite with `PromptbookStorage` and move to `/src/...`
+ */

package/apps/agents-server/src/utils/cdn/utils/getUserFileCdnKey.ts

@@ -0,0 +1,27 @@
+import { titleToName } from '../../../../../../src/utils/normalization/titleToName';
+import hexEncoder from 'crypto-js/enc-hex';
+import sha256 from 'crypto-js/sha256';
+import type { string_uri } from '../../../../../../src/types/typeAliases';
+import { nameToSubfolderPath } from './nameToSubfolderPath';
+
+/**
+ * Generates a path for the user content
+ */
+export function getUserFileCdnKey(file: Buffer, originalFilename: string): string_uri {
+    const hash = sha256(hexEncoder.parse(file.toString('hex'))).toString(/* hex */);
+
+    const originalFilenameParts = originalFilename.split('.');
+    const extension = originalFilenameParts.pop();
+    const name = titleToName(originalFilenameParts.join('.'));
+
+    const filename = name + '.' + extension;
+    // <- Note: [⛳️] Preserving original file name
+
+    return `user/files/${nameToSubfolderPath(hash).join('/')}/${filename}`;
+}
+
+/**
+ * TODO: [🌍] Unite this logic in one place
+ * TODO: Way to garbage unused uploaded files
+ * TODO: Probably separate util countBufferHash
+ */

package/apps/agents-server/src/utils/cdn/utils/nameToSubfolderPath.ts

@@ -0,0 +1,9 @@
+import type { string_name } from '../../../../../../src/types/typeAliases';
+
+export function nameToSubfolderPath(name: string_name): Array<string> {
+    return [name.substr(0, 2).toLowerCase(), name.substr(2, 5).toLowerCase()];
+}
+
+/**
+ * TODO: !!! Use `nameToSubfolderPath` from src
+ */

package/apps/agents-server/src/utils/cdn/utils/nextRequestToNodeRequest.ts

@@ -0,0 +1,27 @@
+import { TODO_any } from '@promptbook-local/types';
+import { NextRequest } from 'next/server';
+import { Readable } from 'node:stream';
+
+export async function nextRequestToNodeRequest(nextRequest: NextRequest): Promise<Readable> {
+    const reader = nextRequest.body?.getReader();
+
+    if (!reader) {
+        throw new Error(`Can not get nextRequest.body.getReader()`);
+    }
+
+    const nodeStream = new Readable({
+        async read() {
+            const { done, value } = await reader.read();
+            if (done) this.push(null);
+            else this.push(Buffer.from(value));
+        },
+    });
+
+    // Fake IncomingMessage with headers
+    (nodeStream as TODO_any).headers = Object.fromEntries(nextRequest.headers.entries());
+    (nodeStream as TODO_any).method = nextRequest.method;
+    (nodeStream as TODO_any).url = nextRequest.url;
+    (nodeStream as TODO_any).socket = {}; // required by formidable
+
+    return nodeStream;
+}

package/apps/agents-server/src/utils/validators/validateMimeType.ts

@@ -0,0 +1,24 @@
+import type { string_mime_type } from '../../../../../src/types/typeAliases';
+
+/**
+ * Checks if the value is valid mime-type
+ *
+ * @param value candidate for mime-type
+ * @returns the value if it is valid mime-type
+ * @throws TypeError if the value is not valid mime-type
+ */
+export function validateMimeType(value: unknown): string_mime_type {
+    if (typeof value !== 'string') {
+        throw new TypeError(`Mime-type must be string, but it is ${typeof value}`);
+    }
+
+    if (!/^[a-z]+\/(?:[a-z0-9]+[.-])*[a-z0-9]+$/i.test(value)) {
+        throw new TypeError(`Invalid mime-type "${value}"`);
+    }
+
+    return value as string_mime_type;
+}
+
+/**
+ * TODO: [🧠] Move to main Promptbook utils
+ */

package/apps/agents-server/tsconfig.json

@@ -24,6 +24,6 @@
             "@promptbook-local/*": ["../../src/_packages/*.index"]
         }
     },
-    "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+    "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts", "../_common/hooks/usePromise.ts"],
     "exclude": ["node_modules"]
 }