@promptbook/cli 0.104.0-1 → 0.104.0-10

package/apps/agents-server/src/message-providers/email/sendgrid/parseInboundSendgridEmail.ts

@@ -0,0 +1,49 @@
+import { string_markdown } from '@promptbook-local/types';
+import { simpleParser } from 'mailparser';
+import TurndownService from 'turndown';
+import { extractBodyContentFromHtml } from '../../../utils/content/extractBodyContentFromHtml';
+import type { InboundEmail } from '../_common/Email';
+import { parseEmailAddress } from '../_common/utils/parseEmailAddress';
+import { parseEmailAddresses } from '../_common/utils/parseEmailAddresses';
+
+/**
+ * Function parseInboundSendgridEmail will parse raw inbound email from Sendgrid and return Email object
+ */
+export async function parseInboundSendgridEmail(rawEmail: string): Promise<InboundEmail> {
+    const parsedEmail = await simpleParser(rawEmail);
+
+    const toArray = !Array.isArray(parsedEmail.to)
+        ? parsedEmail.to === undefined
+            ? []
+            : [parsedEmail.to]
+        : parsedEmail.to;
+    const to = toArray.flatMap((_) => parseEmailAddresses(_.text));
+
+    const ccArray = !Array.isArray(parsedEmail.cc)
+        ? parsedEmail.cc === undefined
+            ? []
+            : [parsedEmail.cc]
+        : parsedEmail.cc;
+    const cc = ccArray.flatMap((_) => parseEmailAddresses(_.text));
+
+    const turndownService = new TurndownService();
+
+    const content = (parsedEmail.html
+        ? turndownService.turndown(extractBodyContentFromHtml(parsedEmail.html))
+        : parsedEmail.text || '') as string_markdown;
+
+    const email: InboundEmail = {
+        channel: 'EMAIL',
+        direction: 'INBOUND',
+        sender: parseEmailAddress(parsedEmail.from?.text || '').fullEmail,
+        recipients: to.map((_) => _.fullEmail),
+        cc,
+        subject: parsedEmail.subject || '',
+        content,
+        attachments: [
+            /* <- TODO: [📯] Parse attachments */
+        ],
+    };
+
+    return email;
+}

package/apps/agents-server/src/message-providers/email/zeptomail/ZeptomailMessageProvider.ts

@@ -0,0 +1,51 @@
+import { removeMarkdownFormatting } from '@promptbook-local/markdown-utils';
+import type { really_any } from '@promptbook-local/types';
+import { marked } from 'marked';
+// @ts-expect-error: Zeptomail types are not resolving correctly
+import { SendMailClient } from 'zeptomail';
+import { MessageProvider } from '../../interfaces/MessageProvider';
+import { OutboundEmail } from '../_common/Email';
+
+export class ZeptomailMessageProvider implements MessageProvider {
+    constructor(private readonly apiKey: string) {}
+
+    public async send(message: OutboundEmail): Promise<really_any> {
+        try {
+            const client = new SendMailClient({ url: 'api.zeptomail.com/', token: this.apiKey });
+
+            const sender = message.sender as really_any;
+            const recipients = (Array.isArray(message.recipients) ? message.recipients : [message.recipients]).filter(
+                Boolean,
+            ) as really_any[];
+
+            const textbody = removeMarkdownFormatting(message.content);
+            const htmlbody = await marked.parse(message.content);
+
+            const response = await client.sendMail({
+                from: {
+                    address: sender.email || sender.baseEmail || sender,
+                    name: sender.name || sender.fullName || undefined,
+                },
+                to: recipients.map((r) => ({
+                    email_address: {
+                        address: r.email || r.baseEmail || r,
+                        name: r.name || r.fullName || undefined,
+                    },
+                })),
+                subject: message.metadata?.subject || 'No Subject',
+                textbody,
+                htmlbody,
+                track_clicks: true,
+                track_opens: true,
+            });
+
+            return response;
+        } catch (raw: really_any) {
+            if (!('error' in raw)) {
+                throw raw;
+            }
+
+            throw new Error(raw.error.message, raw.error.details);
+        }
+    }
+}

package/apps/agents-server/src/message-providers/index.ts

@@ -0,0 +1,13 @@
+import { SendgridMessageProvider } from './email/sendgrid/SendgridMessageProvider';
+import { ZeptomailMessageProvider } from './email/zeptomail/ZeptomailMessageProvider';
+import { MessageProvider } from './interfaces/MessageProvider';
+
+export const EMAIL_PROVIDERS: Record<string, MessageProvider> = {};
+
+if (process.env.ZEPTOMAIL_API_KEY) {
+    EMAIL_PROVIDERS['ZEPTOMAIL'] = new ZeptomailMessageProvider(process.env.ZEPTOMAIL_API_KEY);
+}
+
+if (process.env.SENDGRID_API_KEY) {
+    EMAIL_PROVIDERS['SENDGRID'] = new SendgridMessageProvider(process.env.SENDGRID_API_KEY);
+}

package/apps/agents-server/src/message-providers/interfaces/MessageProvider.ts

@@ -0,0 +1,11 @@
+import type { Message, really_any, string_email } from '@promptbook-local/types';
+
+export type MessageProvider = {
+    /**
+     * Sends a message through the provider
+     *
+     * @param message The message to send
+     * @returns Raw response from the provider
+     */
+    send(message: Message<string_email>): Promise<really_any>;
+};

package/apps/agents-server/src/middleware.ts

@@ -1,13 +1,15 @@
 import { TODO_any } from '@promptbook-local/types';
 import { createClient } from '@supabase/supabase-js';
 import { NextRequest, NextResponse } from 'next/server';
-import { SERVERS, SUPABASE_TABLE_PREFIX } from '../config';
+import { SERVERS } from '../config';
+import { $getTableName } from './database/$getTableName';
+import { RESERVED_PATHS } from './generated/reservedPaths';
 import { isIpAllowed } from './utils/isIpAllowed';
 
-// Note: Re-implementing normalizeTo_PascalCase to avoid importing from @promptbook-local/utils which might have Node.js dependencies
+// Note: Re-implementing normalizeTo_PascalCase to avoid importing from @promptbook-local/utils which might have Node.js dependencies !!!!
 function normalizeTo_PascalCase(text: string): string {
     return text
-        .replace(/(?:^\w|[A-Z]|\b\w)/g, (word, index) => {
+        .replace(/(?:^\w|[A-Z]|\b\w)/g, (word) => {
             return word.toUpperCase();
         })
         .replace(/\s+/g, '');
@@ -33,8 +35,12 @@ export async function middleware(req: NextRequest) {
     const host = req.headers.get('host');
 
     if (host) {
+        /*
+        Note: [🐔] This code was commented out because results of it are unused
+
         let tablePrefix = SUPABASE_TABLE_PREFIX;
 
+        
         if (SERVERS && SERVERS.length > 0) {
             // Logic mirrored from src/tools/$provideServer.ts
             if (SERVERS.some((server) => server === host)) {
@@ -44,6 +50,7 @@ export async function middleware(req: NextRequest) {
                 tablePrefix = `server_${serverName}_`;
             }
         }
+        */
 
         const supabaseUrl = process.env.NEXT_PUBLIC_SUPABASE_URL;
         const supabaseKey = process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY;
@@ -58,7 +65,7 @@ export async function middleware(req: NextRequest) {
                 });
 
                 const { data } = await supabase
-                    .from(`${tablePrefix}Metadata`)
+                    .from(await $getTableName(`Metadata`))
                     .select('value')
                     .eq('key', 'RESTRICT_IP')
                     .single();
@@ -82,6 +89,9 @@ export async function middleware(req: NextRequest) {
         const token = authHeader.split(' ')[1];
 
         if (token.startsWith('ptbk_')) {
+            /*
+            Note: [🐔] This code was commented out because results of it are unused
+            
             const host = req.headers.get('host');
             let tablePrefix = SUPABASE_TABLE_PREFIX;
 
@@ -93,6 +103,7 @@ export async function middleware(req: NextRequest) {
                     tablePrefix = `server_${serverName}_`;
                 }
             }
+            */
 
             const supabaseUrl = process.env.NEXT_PUBLIC_SUPABASE_URL;
             const supabaseKey = process.env.SUPABASE_SERVICE_ROLE_KEY || process.env.NEXT_PUBLIC_SUPABASE_ANON_KEY;
@@ -107,7 +118,7 @@ export async function middleware(req: NextRequest) {
                     });
 
                     const { data } = await supabase
-                        .from(`${tablePrefix}ApiTokens`)
+                        .from(await $getTableName(`ApiTokens`))
                         .select('id')
                         .eq('token', token)
                         .eq('isRevoked', false)
@@ -186,22 +197,7 @@ export async function middleware(req: NextRequest) {
 
     if (
         potentialAgentName &&
-        ![
-            'agents',
-            'api',
-            'admin',
-            'docs',
-            'test',
-            'embed',
-            '_next',
-            'manifest.webmanifest',
-            'sw.js',
-            'favicon.ico',
-            'sitemap.xml',
-            'robots.txt',
-            'security.txt',
-            'humans.txt',
-        ].includes(potentialAgentName) &&
+        !RESERVED_PATHS.includes(potentialAgentName) &&
         !potentialAgentName.startsWith('.') &&
         // Note: Other static files are excluded by the matcher configuration below
         true
@@ -243,7 +239,7 @@ export async function middleware(req: NextRequest) {
                 let serverName = serverHost;
                 serverName = serverName.replace(/\.ptbk\.io$/, '');
                 serverName = normalizeTo_PascalCase(serverName);
-                const prefix = `server_${serverName}_`;
+                // const prefix = `server_${serverName}_`;
 
                 // Search for agent with matching META LINK
                 // agentProfile->links is an array of strings
@@ -256,7 +252,7 @@ export async function middleware(req: NextRequest) {
 
                 try {
                     const { data } = await supabase
-                        .from(`${prefix}Agent`)
+                        .from(await $getTableName(`Agent`))
                         .select('agentName')
                         .or(orFilter)
                         .limit(1)

package/apps/agents-server/src/tools/$provideBrowserForServer.ts

@@ -0,0 +1,32 @@
+import { locateChrome } from 'locate-app';
+import { join } from 'path';
+import { BrowserContext, chromium } from 'playwright';
+
+/**
+ * Cache of browser instance
+ *
+ * @private internal cache for `$provideBrowserForServer`
+ */
+let browserInstance: BrowserContext | null = null;
+
+/**
+ * @@@
+ */
+export async function $provideBrowserForServer(): Promise<BrowserContext> {
+    if (browserInstance !== null && browserInstance.browser() && browserInstance.browser()!.isConnected()) {
+        return browserInstance;
+    }
+
+    console.log('Launching new browser instance...');
+    browserInstance = await chromium.launchPersistentContext(
+        join(process.cwd(), '.promptbook', 'puppeteer', 'user-data'),
+        {
+            executablePath: await locateChrome(),
+            headless: false,
+            // defaultViewport: null,
+            // downloadsPath
+        },
+    );
+
+    return browserInstance;
+}

package/apps/agents-server/src/tools/$provideCdnForServer.ts

@@ -1,3 +1,5 @@
+import { $provideSupabaseForServer } from '../database/$provideSupabaseForServer';
+import { TrackedFilesStorage } from '../utils/cdn/classes/TrackedFilesStorage';
 import { VercelBlobStorage } from '../utils/cdn/classes/VercelBlobStorage';
 import { IIFilesStorageWithCdn } from '../utils/cdn/interfaces/IFilesStorage';
 
@@ -9,16 +11,19 @@ import { IIFilesStorageWithCdn } from '../utils/cdn/interfaces/IFilesStorage';
 let cdn: IIFilesStorageWithCdn | null = null;
 
 /**
- * [🐱‍🚀]
+ * @@@
  */
 export function $provideCdnForServer(): IIFilesStorageWithCdn {
     if (!cdn) {
-        cdn = new VercelBlobStorage({
+        const inner = new VercelBlobStorage({
             token: process.env.VERCEL_BLOB_READ_WRITE_TOKEN!,
             pathPrefix: process.env.NEXT_PUBLIC_CDN_PATH_PREFIX!,
             cdnPublicUrl: new URL(process.env.NEXT_PUBLIC_CDN_PUBLIC_URL!),
         });
 
+        const supabase = $provideSupabaseForServer();
+        cdn = new TrackedFilesStorage(inner, supabase);
+
         /*
         cdn = new DigitalOceanSpaces({
             bucket: process.env.CDN_BUCKET!,

package/apps/agents-server/src/utils/auth.ts

@@ -1,33 +1,133 @@
-import { randomBytes, scrypt, timingSafeEqual } from 'crypto';
+import { createHash, randomBytes, scrypt, timingSafeEqual } from 'crypto';
 import { promisify } from 'util';
+import { PASSWORD_SECURITY_CONFIG } from '../../../../security.config';
 
 const scryptAsync = promisify(scrypt);
 
 /**
- * Hashes a password using scrypt
- * 
- * @param password The plain text password
+ * Validates password input to prevent edge cases and DoS attacks
+ *
+ * @param password The password to validate
+ * @throws Error if password is invalid
+ */
+function validatePasswordInput(password: string): void {
+    if (typeof password !== 'string') {
+        throw new Error('Password must be a string');
+    }
+    if (password.length === 0) {
+        throw new Error('Password cannot be empty');
+    }
+    if (password.length < PASSWORD_SECURITY_CONFIG.MIN_PASSWORD_LENGTH) {
+        throw new Error(`Password must be at least ${PASSWORD_SECURITY_CONFIG.MIN_PASSWORD_LENGTH} characters`);
+    }
+    // Note: No hard max limit - long passwords are compacted via compactPassword()
+}
+
+/**
+ * Compacts a password for secure processing
+ *
+ * If the password is within MAX_PASSWORD_LENGTH, it is returned as-is.
+ * If longer, the password is split at MAX_PASSWORD_LENGTH and the second part
+ * is hashed with SHA256 before being appended to the first part.
+ *
+ * This prevents DoS attacks via extremely long passwords while still utilizing
+ * the full entropy of longer passwords.
+ *
+ * @param password The password to compact
+ * @returns The compacted password
+ */
+function compactPassword(password: string): string {
+    if (password.length <= PASSWORD_SECURITY_CONFIG.MAX_PASSWORD_LENGTH) {
+        return password;
+    }
+
+    const firstPart = password.slice(0, PASSWORD_SECURITY_CONFIG.MAX_PASSWORD_LENGTH);
+    const secondPart = password.slice(PASSWORD_SECURITY_CONFIG.MAX_PASSWORD_LENGTH);
+
+    // Hash the overflow part with SHA256 to bound its length while preserving entropy
+    const secondPartHash = createHash('sha256').update(secondPart, 'utf8').digest('hex');
+
+    return firstPart + secondPartHash;
+}
+
+/**
+ * Hashes a password using scrypt with secure parameters
+ *
+ * @param password The plain text password (minimum 8 characters, no maximum - long passwords are compacted)
  * @returns The salt and hash formatted as "salt:hash"
+ * @throws Error if password validation fails
  */
 export async function hashPassword(password: string): Promise<string> {
-    const salt = randomBytes(16).toString('hex');
-    const derivedKey = (await scryptAsync(password, salt, 64)) as Buffer;
+    validatePasswordInput(password);
+
+    // Compact long passwords to prevent DoS while preserving entropy
+    const compactedPassword = compactPassword(password);
+
+    const salt = randomBytes(PASSWORD_SECURITY_CONFIG.SALT_LENGTH).toString('hex');
+    const derivedKey = (await scryptAsync(compactedPassword, salt, PASSWORD_SECURITY_CONFIG.KEY_LENGTH)) as Buffer;
+
+    // Clear password from memory as soon as possible (best effort)
+    // Note: JavaScript strings are immutable, so this is limited in effectiveness
     return `${salt}:${derivedKey.toString('hex')}`;
 }
 
 /**
- * Verifies a password against a stored hash
- * 
- * @param password The plain text password
+ * Verifies a password against a stored hash using constant-time comparison
+ *
+ * @param password The plain text password to verify
  * @param storedHash The stored hash in format "salt:hash"
- * @returns True if the password matches
+ * @returns True if the password matches, false otherwise
  */
 export async function verifyPassword(password: string, storedHash: string): Promise<boolean> {
-    const [salt, key] = storedHash.split(':');
-    if (!salt || !key) return false;
-    
-    const derivedKey = (await scryptAsync(password, salt, 64)) as Buffer;
-    const keyBuffer = Buffer.from(key, 'hex');
-    
-    return timingSafeEqual(derivedKey, keyBuffer);
+    // Validate inputs
+    if (typeof password !== 'string' || typeof storedHash !== 'string') {
+        return false;
+    }
+
+    if (password.length === 0) {
+        return false;
+    }
+
+    // Compact long passwords the same way as during hashing
+    const compactedPassword = compactPassword(password);
+
+    const parts = storedHash.split(':');
+    if (parts.length !== 2) {
+        return false;
+    }
+
+    const [salt, key] = parts;
+    if (!salt || !key) {
+        return false;
+    }
+
+    // Validate salt and key format (should be hex strings of expected length)
+    const expectedSaltLength = PASSWORD_SECURITY_CONFIG.SALT_LENGTH * 2; // hex encoding doubles length
+    const expectedKeyLength = PASSWORD_SECURITY_CONFIG.KEY_LENGTH * 2;
+
+    if (salt.length !== expectedSaltLength || key.length !== expectedKeyLength) {
+        return false;
+    }
+
+    // Validate hex format
+    const hexRegex = /^[0-9a-fA-F]+$/;
+    if (!hexRegex.test(salt) || !hexRegex.test(key)) {
+        return false;
+    }
+
+    try {
+        const derivedKey = (await scryptAsync(compactedPassword, salt, PASSWORD_SECURITY_CONFIG.KEY_LENGTH)) as Buffer;
+        const keyBuffer = Buffer.from(key, 'hex');
+
+        // Ensure buffers are same length before timing-safe comparison
+        // This should always be true given our validation, but defense in depth
+        if (derivedKey.length !== keyBuffer.length) {
+            return false;
+        }
+
+        return timingSafeEqual(derivedKey, keyBuffer);
+    } catch {
+        // Any error during verification should return false, not leak information
+        return false;
+    }
 }

package/apps/agents-server/src/utils/cdn/classes/TrackedFilesStorage.ts

@@ -0,0 +1,57 @@
+import { SupabaseClient } from '@supabase/supabase-js';
+import { $getTableName } from '../../../database/$getTableName';
+import { AgentsServerDatabase } from '../../../database/schema';
+import type { IFile, IIFilesStorageWithCdn } from '../interfaces/IFilesStorage';
+
+export class TrackedFilesStorage implements IIFilesStorageWithCdn {
+    public constructor(
+        private readonly inner: IIFilesStorageWithCdn,
+        private readonly supabase: SupabaseClient<AgentsServerDatabase>,
+    ) {}
+
+    public get cdnPublicUrl(): URL {
+        return this.inner.cdnPublicUrl;
+    }
+
+    public get pathPrefix(): string | undefined {
+        return this.inner.pathPrefix;
+    }
+
+    public getItemUrl(key: string): URL {
+        return this.inner.getItemUrl(key);
+    }
+
+    public async getItem(key: string): Promise<IFile | null> {
+        return this.inner.getItem(key);
+    }
+
+    public async removeItem(key: string): Promise<void> {
+        return this.inner.removeItem(key);
+    }
+
+    public async setItem(key: string, file: IFile): Promise<void> {
+        console.log('!!! 0', { key, file });
+
+        await this.inner.setItem(key, file);
+
+        try {
+            const { userId, purpose } = file;
+            const cdnUrl = this.getItemUrl(key).href;
+
+            console.log('!!! 1', { userId, purpose, cdnUrl, key, file });
+
+            await this.supabase.from(await $getTableName('File')).insert({
+                userId: userId || null,
+                fileName: key,
+                fileSize: file.fileSize ?? file.data.length,
+                fileType: file.type,
+                cdnUrl,
+                purpose: purpose || 'UNKNOWN',
+            });
+
+            console.log('!!! 2', { userId, purpose, cdnUrl, key, file });
+        } catch (error) {
+            console.error('Failed to track upload:', error);
+        }
+    }
+}

package/apps/agents-server/src/utils/cdn/classes/VercelBlobStorage.ts

@@ -14,6 +14,10 @@ export class VercelBlobStorage implements IIFilesStorageWithCdn {
         return this.config.cdnPublicUrl;
     }
 
+    public get pathPrefix() {
+        return this.config.pathPrefix;
+    }
+
     public constructor(private readonly config: IVercelBlobStorageConfig) {}
 
     public getItemUrl(key: string): URL {

package/apps/agents-server/src/utils/cdn/interfaces/IFilesStorage.ts

@@ -5,6 +5,23 @@ export type IFile = {
     // Maybe TODO name: string_name;
     type: string_mime_type;
     data: Buffer;
+
+    /**
+     * User who uploaded the file
+     */
+    userId?: number;
+
+    /**
+     * Purpose of the upload (e.g. KNOWLEDGE, SERVER_FAVICON_URL)
+     */
+    purpose?: string;
+
+    /**
+     * Size of the file in bytes
+     *
+     * Note: This is optional, if not provided, the size of the buffer is used
+     */
+    fileSize?: number;
 };
 
 /**
@@ -17,6 +34,7 @@ export type IFilesStorage = Omit<IStorage<IFile>, 'length' | 'clear' | 'key'>;
  */
 export type IIFilesStorageWithCdn = IFilesStorage & {
     readonly cdnPublicUrl: URL;
+    readonly pathPrefix?: string;
     getItemUrl(key: string): URL;
 };
 

package/apps/agents-server/src/utils/content/extractBodyContentFromHtml.ts

@@ -0,0 +1,19 @@
+import type { string_html } from '@promptbook-local/types';
+
+/**
+ * Extract the first heading from HTML
+ *
+ * @param contentText HTML
+ * @returns heading
+ */
+export function extractBodyContentFromHtml(html: string_html): string_html {
+    // Note: Not using DOMParser, because it's overkill for this simple task
+
+    const match = html.match(/<body[^>]*>(?<bodyContent>[\s\S]*)<\/body>/s);
+
+    if (!match) {
+        return html;
+    }
+
+    return match.groups!.bodyContent!;
+}

package/apps/agents-server/src/utils/getUserIdFromRequest.ts

@@ -0,0 +1,35 @@
+import { NextRequest } from 'next/server';
+import { keepUnused } from '../../../../src/utils/organization/keepUnused';
+import { $getTableName } from '../database/$getTableName';
+import { $provideSupabaseForServer } from '../database/$provideSupabaseForServer';
+import { getSession } from './session';
+
+export async function getUserIdFromRequest(request: NextRequest): Promise<number | null> {
+    keepUnused(request); // Unused because we get user from session cookie for now
+
+    try {
+        // 1. Try to get user from session (cookie)
+        const session = await getSession();
+        if (session && session.username) {
+            const supabase = $provideSupabaseForServer();
+            const { data } = await supabase
+                .from(await $getTableName('User'))
+                .select('id')
+                .eq('username', session.username)
+                .single();
+
+            if (data) {
+                return data.id;
+            }
+        }
+
+        // 2. Try to get user from API key (Authorization header)
+        // TODO: [🧠] Implement linking API keys to users if needed
+        // const authHeader = request.headers.get('authorization');
+        // ...
+    } catch (error) {
+        console.error('Error getting user ID from request:', error);
+    }
+
+    return null;
+}