@promptbook/cli 0.112.0-125 → 0.112.0-127

package/apps/agents-server/src/utils/chat/resolveAgentsServerCitationLabel.ts

@@ -0,0 +1,130 @@
+'use client';
+
+import type { ChatProps } from '@promptbook-local/components';
+
+/**
+ * Citation label resolver type accepted by the shared Chat component.
+ *
+ * @private utility type of Agents Server chat citations
+ */
+type AgentsServerCitationLabelResolver = NonNullable<ChatProps['resolveCitationLabel']>;
+
+/**
+ * Cache of in-flight and completed citation label lookups.
+ *
+ * @private utility constant of Agents Server chat citations
+ */
+const citationLabelCache = new Map<string, Promise<string | null>>();
+
+/**
+ * Pattern matching HTTP(S) URLs that the server can fetch.
+ *
+ * @private utility constant of Agents Server chat citations
+ */
+const HTTP_URL_REGEX = /^https?:\/\//i;
+
+/**
+ * Resolves nicer citation labels for Agents Server chat sources.
+ *
+ * @private utility of Agents Server chat citations
+ */
+export const resolveAgentsServerCitationLabel: AgentsServerCitationLabelResolver = async (citation) => {
+    const title = normalizeCitationLabelResponse(citation.title);
+    if (title) {
+        return title;
+    }
+
+    if (!hasResolvableCitationTarget(citation.source, citation.url)) {
+        return null;
+    }
+
+    const cacheKey = createCitationLabelCacheKey(citation.source, citation.url, citation.title);
+    const cachedLabel = citationLabelCache.get(cacheKey);
+    if (cachedLabel) {
+        return cachedLabel;
+    }
+
+    const labelPromise = requestCitationLabel({
+        source: citation.source,
+        url: citation.url,
+        title: citation.title,
+    });
+    citationLabelCache.set(cacheKey, labelPromise);
+
+    return labelPromise;
+};
+
+/**
+ * Requests one citation label from the server.
+ *
+ * @param payload - Citation metadata sent to the server resolver.
+ * @returns Resolved label or null.
+ *
+ * @private utility of Agents Server chat citations
+ */
+async function requestCitationLabel(payload: {
+    readonly source: string;
+    readonly url?: string;
+    readonly title?: string;
+}): Promise<string | null> {
+    const response = await fetch('/api/chat/citation-label', {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+        },
+        body: JSON.stringify(payload),
+    }).catch(() => null);
+
+    if (!response?.ok) {
+        return null;
+    }
+
+    const body = (await response.json().catch(() => null)) as { label?: unknown } | null;
+
+    return normalizeCitationLabelResponse(body?.label);
+}
+
+/**
+ * Creates a stable cache key for one citation lookup.
+ *
+ * @param source - Citation source.
+ * @param url - Optional resolved citation URL.
+ * @param title - Optional title.
+ * @returns Cache key.
+ *
+ * @private utility of Agents Server chat citations
+ */
+function createCitationLabelCacheKey(source: string, url?: string, title?: string): string {
+    return [source, url || '', title || ''].join('\n');
+}
+
+/**
+ * Checks whether a citation has a URL target worth resolving on the server.
+ *
+ * @param source - Citation source.
+ * @param url - Optional resolved citation URL.
+ * @returns True when the server can attempt a metadata lookup.
+ *
+ * @private utility of Agents Server chat citations
+ */
+function hasResolvableCitationTarget(source: string, url?: string): boolean {
+    return HTTP_URL_REGEX.test(url || '') || HTTP_URL_REGEX.test(source);
+}
+
+/**
+ * Normalizes one label value received from the server or structured citation metadata.
+ *
+ * @param value - Raw label value.
+ * @returns Trimmed label or null.
+ *
+ * @private utility of Agents Server chat citations
+ */
+function normalizeCitationLabelResponse(value: unknown): string | null {
+    if (typeof value !== 'string') {
+        return null;
+    }
+
+    const label = value.replace(/\s+/g, ' ').trim();
+
+    return label || null;
+}

package/apps/agents-server/src/utils/chat/resolveCitationSourceLabel.ts

@@ -0,0 +1,436 @@
+import { JSDOM } from 'jsdom';
+
+/**
+ * Citation metadata accepted by the Agents Server source-label resolver.
+ *
+ * @private utility type of Agents Server chat citation labels
+ */
+export type CitationSourceLabelPayload = {
+    /**
+     * Raw citation source from the model response.
+     */
+    readonly source: string;
+
+    /**
+     * Optional resolved URL for the citation source.
+     */
+    readonly url?: string;
+
+    /**
+     * Optional title already attached to the citation.
+     */
+    readonly title?: string;
+};
+
+/**
+ * Maximum number of bytes fetched when resolving one citation title.
+ *
+ * @private utility constant of Agents Server chat citation labels
+ */
+const MAX_SOURCE_LABEL_BYTES = 1_000_000;
+
+/**
+ * Timeout for metadata fetches so the chat UI is not blocked by slow sources.
+ *
+ * @private utility constant of Agents Server chat citation labels
+ */
+const SOURCE_LABEL_FETCH_TIMEOUT_MS = 4_000;
+
+/**
+ * User agent used for lightweight source metadata requests.
+ *
+ * @private utility constant of Agents Server chat citation labels
+ */
+const SOURCE_LABEL_USER_AGENT = 'Promptbook Agents Server citation label resolver';
+
+/**
+ * Pattern matching markdown headings.
+ *
+ * @private utility constant of Agents Server chat citation labels
+ */
+const MARKDOWN_HEADING_REGEX = /^\s{0,3}#{1,6}\s+(.+?)\s*#*\s*$/m;
+
+/**
+ * Pattern matching common PDF/XMP title metadata.
+ *
+ * @private utility constant of Agents Server chat citation labels
+ */
+const PDF_XMP_TITLE_REGEX = /<dc:title\b[^>]*>[\s\S]*?<rdf:li\b[^>]*>([\s\S]*?)<\/rdf:li>[\s\S]*?<\/dc:title>/i;
+
+/**
+ * Pattern matching simple PDF/XMP title metadata without `rdf:li`.
+ *
+ * @private utility constant of Agents Server chat citation labels
+ */
+const PDF_XMP_SIMPLE_TITLE_REGEX = /<dc:title\b[^>]*>([\s\S]*?)<\/dc:title>/i;
+
+/**
+ * Pattern matching PDF literal title metadata.
+ *
+ * @private utility constant of Agents Server chat citation labels
+ */
+const PDF_LITERAL_TITLE_REGEX = /\/Title\s*\(((?:\\.|[^\\)]){1,500})\)/s;
+
+/**
+ * Pattern matching PDF hexadecimal title metadata.
+ *
+ * @private utility constant of Agents Server chat citation labels
+ */
+const PDF_HEX_TITLE_REGEX = /\/Title\s*<([0-9a-fA-F\s]{2,1000})>/;
+
+/**
+ * Returns the HTTP(S) URL that can be fetched for richer citation metadata.
+ *
+ * @param citation - Citation payload received from the chat UI.
+ * @returns URL string or null when the citation has no fetchable target.
+ *
+ * @private utility of Agents Server chat citation labels
+ */
+export function resolveCitationLabelTargetUrl(citation: CitationSourceLabelPayload): string | null {
+    return normalizeHttpUrl(citation.url) || normalizeHttpUrl(citation.source);
+}
+
+/**
+ * Resolves a human-readable source label from page or document metadata.
+ *
+ * @param citation - Citation metadata from the chat UI.
+ * @returns Resolved label or null when no useful title metadata is available.
+ *
+ * @private utility of Agents Server chat citation labels
+ */
+export async function resolveCitationSourceLabel(citation: CitationSourceLabelPayload): Promise<string | null> {
+    const explicitTitle = normalizeResolvedCitationLabel(citation.title);
+    if (explicitTitle) {
+        return explicitTitle;
+    }
+
+    const targetUrl = resolveCitationLabelTargetUrl(citation);
+    if (!targetUrl) {
+        return null;
+    }
+
+    const response = await fetch(targetUrl, {
+        headers: {
+            'User-Agent': SOURCE_LABEL_USER_AGENT,
+            Accept: 'text/html,application/xhtml+xml,application/pdf,text/markdown,text/plain;q=0.9,*/*;q=0.1',
+        },
+        signal: AbortSignal.timeout(SOURCE_LABEL_FETCH_TIMEOUT_MS),
+    });
+
+    if (!response.ok) {
+        return null;
+    }
+
+    const bytes = await readResponseSnippet(response, MAX_SOURCE_LABEL_BYTES);
+    const contentType = response.headers.get('content-type')?.toLowerCase() || '';
+    const isPdf = contentType.includes('application/pdf') || targetUrl.toLowerCase().split('?')[0]?.endsWith('.pdf');
+
+    if (isPdf || startsWithPdfSignature(bytes)) {
+        return extractCitationLabelFromPdfBytes(bytes);
+    }
+
+    const text = new TextDecoder('utf-8', { fatal: false }).decode(bytes);
+    if (contentType.includes('markdown') || contentType.includes('text/plain')) {
+        return extractCitationLabelFromPlainText(text);
+    }
+
+    return extractCitationLabelFromHtml(text) || extractCitationLabelFromPlainText(text);
+}
+
+/**
+ * Extracts a title from an HTML document.
+ *
+ * @param html - HTML source snippet.
+ * @returns Metadata title, heading, or null.
+ *
+ * @private utility of Agents Server chat citation labels
+ */
+export function extractCitationLabelFromHtml(html: string): string | null {
+    const dom = new JSDOM(html);
+    const document = dom.window.document;
+    const candidates = [
+        getMetaContent(document, 'property', 'og:title'),
+        getMetaContent(document, 'name', 'twitter:title'),
+        document.querySelector('title')?.textContent,
+        document.querySelector('h1')?.textContent,
+    ];
+
+    return candidates.map(normalizeResolvedCitationLabel).find((label): label is string => Boolean(label)) || null;
+}
+
+/**
+ * Extracts a title from plain text or markdown-like document snippets.
+ *
+ * @param text - Text source snippet.
+ * @returns Heading or first useful line.
+ *
+ * @private utility of Agents Server chat citation labels
+ */
+export function extractCitationLabelFromPlainText(text: string): string | null {
+    const headingMatch = text.match(MARKDOWN_HEADING_REGEX);
+    const heading = normalizeResolvedCitationLabel(headingMatch?.[1]);
+    if (heading) {
+        return heading;
+    }
+
+    const firstLine = text
+        .split(/\r?\n/)
+        .map(normalizeResolvedCitationLabel)
+        .find((line): line is string => Boolean(line));
+
+    return firstLine || null;
+}
+
+/**
+ * Extracts a title from PDF metadata bytes.
+ *
+ * @param bytes - PDF byte snippet.
+ * @returns PDF metadata title or null.
+ *
+ * @private utility of Agents Server chat citation labels
+ */
+export function extractCitationLabelFromPdfBytes(bytes: Uint8Array): string | null {
+    const latin1Text = Buffer.from(bytes).toString('latin1');
+    const utf8Text = new TextDecoder('utf-8', { fatal: false }).decode(bytes);
+    const xmpTitle = extractPdfXmpTitle(utf8Text);
+    if (xmpTitle) {
+        return xmpTitle;
+    }
+
+    const literalTitleMatch = latin1Text.match(PDF_LITERAL_TITLE_REGEX);
+    const literalTitle = normalizeResolvedCitationLabel(
+        literalTitleMatch?.[1] ? decodePdfLiteralString(literalTitleMatch[1]) : undefined,
+    );
+    if (literalTitle) {
+        return literalTitle;
+    }
+
+    const hexTitleMatch = latin1Text.match(PDF_HEX_TITLE_REGEX);
+    return normalizeResolvedCitationLabel(hexTitleMatch?.[1] ? decodePdfHexString(hexTitleMatch[1]) : undefined);
+}
+
+/**
+ * Reads at most `maxBytes` bytes from one fetch response.
+ *
+ * @param response - Fetch response.
+ * @param maxBytes - Maximum number of bytes to keep.
+ * @returns Collected response bytes.
+ *
+ * @private utility of Agents Server chat citation labels
+ */
+async function readResponseSnippet(response: Response, maxBytes: number): Promise<Uint8Array> {
+    if (!response.body) {
+        return new Uint8Array(await response.arrayBuffer()).slice(0, maxBytes);
+    }
+
+    const reader = response.body.getReader();
+    const chunks: Uint8Array[] = [];
+    let collectedBytes = 0;
+
+    try {
+        while (collectedBytes < maxBytes) {
+            const { done, value } = await reader.read();
+            if (done || !value) {
+                break;
+            }
+
+            const remainingBytes = maxBytes - collectedBytes;
+            const chunk = value.length > remainingBytes ? value.slice(0, remainingBytes) : value;
+            chunks.push(chunk);
+            collectedBytes += chunk.length;
+        }
+
+        if (collectedBytes >= maxBytes) {
+            await reader.cancel();
+        }
+    } finally {
+        reader.releaseLock();
+    }
+
+    const bytes = new Uint8Array(collectedBytes);
+    let offset = 0;
+    for (const chunk of chunks) {
+        bytes.set(chunk, offset);
+        offset += chunk.length;
+    }
+
+    return bytes;
+}
+
+/**
+ * Extracts the content attribute from a matching metadata tag.
+ *
+ * @param document - Parsed DOM document.
+ * @param attribute - Metadata selector attribute.
+ * @param value - Metadata selector value.
+ * @returns Metadata content or undefined.
+ *
+ * @private utility of Agents Server chat citation labels
+ */
+function getMetaContent(document: Document, attribute: 'name' | 'property', value: string): string | undefined {
+    return document.querySelector(`meta[${attribute}="${value}"]`)?.getAttribute('content') || undefined;
+}
+
+/**
+ * Normalizes a title candidate into a compact display label.
+ *
+ * @param label - Raw label candidate.
+ * @returns Trimmed label or null.
+ *
+ * @private utility of Agents Server chat citation labels
+ */
+function normalizeResolvedCitationLabel(label: string | null | undefined): string | null {
+    const normalized = (label || '').replace(/\s+/g, ' ').trim();
+
+    return normalized || null;
+}
+
+/**
+ * Returns a normalized HTTP(S) URL candidate.
+ *
+ * @param value - URL candidate.
+ * @returns URL href or null.
+ *
+ * @private utility of Agents Server chat citation labels
+ */
+function normalizeHttpUrl(value: string | undefined): string | null {
+    if (!value) {
+        return null;
+    }
+
+    try {
+        const url = new URL(value.trim());
+        return url.protocol === 'http:' || url.protocol === 'https:' ? url.href : null;
+    } catch {
+        return null;
+    }
+}
+
+/**
+ * Checks whether bytes start with a PDF file signature.
+ *
+ * @param bytes - Response bytes.
+ * @returns True when the snippet looks like a PDF.
+ *
+ * @private utility of Agents Server chat citation labels
+ */
+function startsWithPdfSignature(bytes: Uint8Array): boolean {
+    return bytes.length >= 4 && bytes[0] === 0x25 && bytes[1] === 0x50 && bytes[2] === 0x44 && bytes[3] === 0x46;
+}
+
+/**
+ * Extracts an XMP title from a PDF text snippet.
+ *
+ * @param text - UTF-8 decoded PDF snippet.
+ * @returns XMP title or null.
+ *
+ * @private utility of Agents Server chat citation labels
+ */
+function extractPdfXmpTitle(text: string): string | null {
+    const title =
+        text.match(PDF_XMP_TITLE_REGEX)?.[1] || text.match(PDF_XMP_SIMPLE_TITLE_REGEX)?.[1]?.replace(/<[^>]+>/g, ' ');
+
+    return normalizeResolvedCitationLabel(title ? decodeHtmlText(title) : undefined);
+}
+
+/**
+ * Decodes text entities using the DOM parser already used for HTML titles.
+ *
+ * @param value - Raw HTML text.
+ * @returns Decoded text.
+ *
+ * @private utility of Agents Server chat citation labels
+ */
+function decodeHtmlText(value: string): string {
+    return new JSDOM(`<span>${value}</span>`).window.document.querySelector('span')?.textContent || value;
+}
+
+/**
+ * Decodes a PDF literal string.
+ *
+ * @param value - Escaped PDF literal value.
+ * @returns Decoded text.
+ *
+ * @private utility of Agents Server chat citation labels
+ */
+function decodePdfLiteralString(value: string): string {
+    const decoded = value
+        .replace(/\\\r?\n/g, '')
+        .replace(/\\([nrtbf()\\])/g, (_match, escaped: string) => {
+            const replacements: Record<string, string> = {
+                n: '\n',
+                r: '\r',
+                t: '\t',
+                b: '\b',
+                f: '\f',
+                '(': '(',
+                ')': ')',
+                '\\': '\\',
+            };
+
+            return replacements[escaped] || escaped;
+        })
+        .replace(/\\([0-7]{1,3})/g, (_match, octal: string) => String.fromCharCode(parseInt(octal, 8)));
+
+    return decodePdfBinaryString(decoded);
+}
+
+/**
+ * Decodes a PDF hexadecimal string.
+ *
+ * @param value - Hex encoded PDF string.
+ * @returns Decoded text.
+ *
+ * @private utility of Agents Server chat citation labels
+ */
+function decodePdfHexString(value: string): string {
+    const normalizedHex = value.replace(/\s+/g, '');
+    const evenHex = normalizedHex.length % 2 === 0 ? normalizedHex : `${normalizedHex}0`;
+    const bytes = new Uint8Array(evenHex.length / 2);
+
+    for (let index = 0; index < evenHex.length; index += 2) {
+        bytes[index / 2] = parseInt(evenHex.slice(index, index + 2), 16);
+    }
+
+    if (bytes[0] === 0xfe && bytes[1] === 0xff) {
+        return decodeUtf16BeBytes(bytes.slice(2));
+    }
+
+    return new TextDecoder('utf-8', { fatal: false }).decode(bytes);
+}
+
+/**
+ * Decodes a binary string that may use a UTF-16BE byte-order mark.
+ *
+ * @param value - Binary string with one byte per character.
+ * @returns Decoded text.
+ *
+ * @private utility of Agents Server chat citation labels
+ */
+function decodePdfBinaryString(value: string): string {
+    if (value.charCodeAt(0) === 0xfe && value.charCodeAt(1) === 0xff) {
+        const bytes = Uint8Array.from(value.slice(2), (character) => character.charCodeAt(0));
+        return decodeUtf16BeBytes(bytes);
+    }
+
+    return value;
+}
+
+/**
+ * Decodes UTF-16BE bytes into a JavaScript string.
+ *
+ * @param bytes - Big-endian UTF-16 bytes.
+ * @returns Decoded text.
+ *
+ * @private utility of Agents Server chat citation labels
+ */
+function decodeUtf16BeBytes(bytes: Uint8Array): string {
+    let text = '';
+    for (let index = 0; index + 1 < bytes.length; index += 2) {
+        text += String.fromCharCode((bytes[index]! << 8) | bytes[index + 1]!);
+    }
+
+    return text;
+}
+
+// Note: [🟢] Code for Agents Server chat citation label resolver should never be published into packages that could be imported into browser environment

package/apps/agents-server/src/utils/knowledge/resolveWebsiteKnowledgeSourcesForServer.ts

@@ -1,3 +1,4 @@
+import { spaceTrim } from 'spacetrim';
 import type { string_knowledge_source_link } from '../../../../../src/types/typeAliases';
 import { $provideFilesystemForNode } from '../../../../../src/scrapers/_common/register/$provideFilesystemForNode';
 import type { ScraperSourceHandler } from '../../../../../src/scrapers/_common/Scraper';
@@ -121,7 +122,15 @@ export async function resolveWebsiteKnowledgeSourcesForServer(
                 continue;
             }
 
-            const inlineSource = createInlineKnowledgeSourceFile(`Source URL: ${source}\n\n${markdown}`);
+            const inlineSource = createInlineKnowledgeSourceFile(
+                spaceTrim(
+                    (block) => `
+                        Source URL: ${source}
+
+                        ${block(markdown)}
+                    `,
+                ),
+            );
             const uploadedSource = await uploadInlineKnowledgeSource(inlineSource);
             resolvedKnowledgeSources.push(uploadedSource);
         } catch (error) {

package/apps/agents-server/src/utils/session.ts

@@ -1,6 +1,8 @@
-import { createHmac } from 'crypto';
+import { createHmac, randomBytes } from 'crypto';
 import { cookies, headers } from 'next/headers';
 import { cache } from 'react';
+import { spaceTrim } from 'spacetrim';
+import { EnvironmentMismatchError } from '../../../../src/errors/EnvironmentMismatchError';
 import { isStandaloneVpsRawIpBootstrapActive } from './standaloneVpsRawIpBootstrap';
 
 /**
@@ -9,9 +11,64 @@ import { isStandaloneVpsRawIpBootstrapActive } from './standaloneVpsRawIpBootstr
 export const SESSION_COOKIE_NAME = 'sessionToken';
 
 /**
- * Secret key used for signing session payloads.
+ * Cached HMAC signing key resolved from `SESSION_SECRET` on first use.
+ *
+ * Computed lazily so module imports never fail eagerly at build time and the
+ * environment can still be configured before the first authenticated request.
+ */
+let resolvedSessionSigningKey: string | null = null;
+
+/**
+ * Resolves the HMAC signing key used for session payloads.
+ *
+ * In production, `SESSION_SECRET` must be configured explicitly — falling back
+ * to any shared credential (in particular `ADMIN_PASSWORD`) or a hardcoded
+ * literal would let anyone who learns that value forge session tokens for any
+ * user (including `admin`). Outside production, a per-process random key is
+ * generated on first use so local development works without configuration
+ * while still rejecting the unsafe hardcoded default.
+ *
+ * @returns HMAC signing key.
+ * @throws {EnvironmentMismatchError} When `SESSION_SECRET` is missing in production.
  */
-const SECRET_KEY = process.env.ADMIN_PASSWORD || 'default-secret-key-change-me';
+function getSessionSigningKey(): string {
+    if (resolvedSessionSigningKey !== null) {
+        return resolvedSessionSigningKey;
+    }
+
+    const configuredSecret = process.env.SESSION_SECRET?.trim();
+    if (configuredSecret) {
+        resolvedSessionSigningKey = configuredSecret;
+        return resolvedSessionSigningKey;
+    }
+
+    if (process.env.NODE_ENV === 'production') {
+        throw new EnvironmentMismatchError(
+            spaceTrim(`
+                Missing required \`SESSION_SECRET\` environment variable in production.
+
+                The Agents Server signs session cookies with HMAC-SHA256 keyed by
+                \`SESSION_SECRET\`. Reusing \`ADMIN_PASSWORD\` or a hardcoded fallback
+                would let anyone who learns that value forge session tokens for any
+                user (including \`admin\`).
+
+                **Fix:** set \`SESSION_SECRET\` to a long random string (for example
+                the output of \`openssl rand -hex 32\`) in the deployment environment
+                and restart the server.
+            `),
+        );
+    }
+
+    resolvedSessionSigningKey = randomBytes(32).toString('hex');
+    console.warn(
+        spaceTrim(`
+            \`SESSION_SECRET\` is not configured — generated a random per-process
+            signing key for this non-production run. Existing session cookies will
+            be invalidated on every restart until \`SESSION_SECRET\` is set.
+        `),
+    );
+    return resolvedSessionSigningKey;
+}
 
 /**
  * Signed session payload persisted in the authentication cookie.
@@ -73,7 +130,7 @@ export type SessionCookieSecurityContext = {
  */
 export function serializeSessionToken(user: SessionUser): string {
     const payload = JSON.stringify(user);
-    const signature = createHmac('sha256', SECRET_KEY).update(payload).digest('hex');
+    const signature = createHmac('sha256', getSessionSigningKey()).update(payload).digest('hex');
     return `${Buffer.from(payload).toString('base64')}.${signature}`;
 }
 
@@ -94,7 +151,7 @@ export function parseSessionToken(token: string | null | undefined): SessionUser
     }
 
     const payload = Buffer.from(payloadBase64, 'base64').toString('utf-8');
-    const expectedSignature = createHmac('sha256', SECRET_KEY).update(payload).digest('hex');
+    const expectedSignature = createHmac('sha256', getSessionSigningKey()).update(payload).digest('hex');
 
     if (signature !== expectedSignature) {
         return null;