create-wirejs-app 2.0.168-llm → 2.0.170

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "create-wirejs-app",
-	"version": "2.0.168-llm",
+	"version": "2.0.170",
 	"description": "Initializes a wirejs package.",
 	"author": "Jon Wire",
 	"license": "MIT",

package/templates/default/api/apps/admin.ts

@@ -5,7 +5,7 @@ import {
 	withContext,
 } from "wirejs-resources";
 
-const permanentAdmins = ['admin'];
+const permanentAdmins = ['admin', 'iambipedal@gmail.com'];
 
 export type Setting = {
 	key: string;

package/templates/default/api/apps/llm/index.ts

@@ -0,0 +1,63 @@
+import {
+	AuthenticationApi,
+	BackgroundJob,
+	withContext
+} from "wirejs-resources";
+import { tooledHandler } from "./tooled-handler.js";
+import { Infra } from "./infra.js";
+export type { Chunk, ChunkData, Conversation, ConversationMessage, Role } from './types.js';
+
+export const LLM = (auth: AuthenticationApi) => {
+	const infra = new Infra('app', 'llm', {
+		models: ['mistral-nemo', 'llama3.2', 'llama3:8b', 'llama2'],
+		systemPrompt: `You are a helpful assistant.`,
+	});
+
+	const chatRunner = new BackgroundJob('app', 'chatRunner', {
+		handler: tooledHandler(infra)
+	});
+
+	return withContext(context => ({
+		async send(room: string, message: string) {
+			const user = await auth.requireCurrentUser(context);
+			await infra.assertUserIsAuthorized(user, room);
+			if (!room || !message || !message.trim()) {
+				throw new Error('Room and message are required');
+			}
+			await chatRunner.start(room, message.trim());
+		},
+		async getRoom(room: string) {
+			const user = await auth.requireCurrentUser(context);
+			await infra.assertUserIsAuthorized(user, room);
+			return infra.getStream(context, room);
+		},
+		async getHistory(room: string) {
+			const user = await auth.requireCurrentUser(context);
+			await infra.assertUserIsAuthorized(user, room);
+			const history = await infra.getHistory(room);
+			return history.filter(m => [
+				'assistant', 'user', 'step'
+			].includes(m.role) && !!m.content);
+		},
+		async getMessage(room: string, mid: number) {
+			const user = await auth.requireCurrentUser(context);
+			await infra.assertUserIsAuthorized(user, room);
+			return infra.getMessage(room, mid);
+		},
+		async createRoom() {
+			const user = await auth.requireCurrentUser(context);
+			const conversation = await infra.createConversation(user);
+			return conversation.conversationId;
+		},
+		async getConversations() {
+			const user = await auth.requireCurrentUser(context);
+			return infra.listUserConversations(user);
+		},
+		async deleteConversation(conversationId: string) {
+			const user = await auth.requireCurrentUser(context);
+			await infra.assertUserIsAuthorized(user, conversationId);
+			await infra.deleteConversation(conversationId)
+			return { success: true };
+		}
+	}))
+};

package/templates/default/api/apps/llm/infra.ts

@@ -0,0 +1,322 @@
+import { randomUUID } from 'crypto';
+import {
+	AssistantMessage,
+	Context,
+	DistributedTable,
+	LLM as LLMService,
+	LLMChunk,
+	LLMMessage,
+	PassThruParser,
+	RealtimeService,
+	Resource,
+	Setting,
+	User,
+	ToolDefinition,
+} from "wirejs-resources";
+import { fromAsync, pad } from "./utils.js";
+import { Chunk, ChunkData, Conversation, ConversationMessage, WorkflowStep } from "./types.js";
+
+const DEFAULT_MODELS_LIST = ['mistral-nemo', 'llama3.2', 'llama3:8b', 'llama2'];
+
+export type PromptOptions = {
+	systemPromptOverride?: string;
+	history: LLMMessage[];
+	tools?: ToolDefinition[];
+} | {
+	systemPromptOverride?: string;
+	prompt: string;
+	tools?: ToolDefinition[];
+};
+
+export type RespondOptions = {
+	conversationId: string;
+	history: LLMMessage[];
+	systemPromptOverride?: string;
+	tools?: ToolDefinition[],
+} | {
+	conversationId: string;
+	mid: number;
+	prompt: string;
+	systemPromptOverride?: string;
+	tools?: ToolDefinition[],
+};
+
+export type InfraOptions = {
+	models?: string[];
+	systemPrompt?: string;
+	tools?: ToolDefinition[],
+}
+
+export class Infra extends Resource {
+	private conversations: ReturnType<typeof makeConversationsTable>;
+	private messages: ReturnType<typeof makeMessagesTable>;
+	private realtime: ReturnType<typeof makeRealtimeService>;
+	private llm: ReturnType<typeof makeLLMService>;
+	private modelSetting: ReturnType<typeof makeModelsOverrideSetting>;
+	private tools?: ToolDefinition[];
+
+	constructor(scope: string | Resource, id: string, options?: InfraOptions) {
+		super(scope, id);
+		this.conversations = makeConversationsTable(this);
+		this.messages = makeMessagesTable(this);
+		this.realtime = makeRealtimeService(this);
+		this.llm = makeLLMService(this, options?.systemPrompt, options?.models);
+		this.modelSetting = makeModelsOverrideSetting(this, options?.models);
+	}
+
+	async prompt(options: PromptOptions): Promise<AssistantMessage> {
+		// TODO: debounce and/or redesign model settings relationship
+		const models = (await this.modelSetting.read()).split(',').map(s => s.trim());
+		const tools = options.tools ?? this.tools;
+
+		console.log('prompt', (options as any).prompt);
+
+		return this.llm.continueConversation({
+			systemPrompt: options.systemPromptOverride,
+			history: 'history' in options ? options.history : [{
+				role: 'user',
+				content: options.prompt
+			}],
+			...(models.length > 0 ? { models } : {}),
+			...(tools ? { tools } : {}),
+		});
+	}
+
+	async respond(options: RespondOptions): Promise<ConversationMessage & AssistantMessage> {
+		// TODO: debounce and/or redesign model settings relationship
+		const models = (await this.modelSetting.read()).split(',').map(s => s.trim());
+
+		const mid = 'mid' in options ? options.mid : options.history.length;
+
+		// responses are not stream directly because they're unnecessarily frequent.
+		// we slow this down, send messages in batches to reduce unnecessary cost.
+		let seq = 0;
+		let batch: string[] = [];
+		let lastBatch = new Date().getTime();
+
+		const onChunk = options.conversationId ?
+			(async (chunk: LLMChunk) => {
+				if (chunk.message.role !== 'assistant') return;
+				if ((chunk.message.tool_calls?.length ?? 0) > 0) return;
+				if (chunk.message.content === '') return;
+				batch.push(chunk.message.content);
+				if (new Date().getTime() - lastBatch > 150) {
+					const text = batch.join('');
+					batch = [];
+					await this.realtime.publish(options.conversationId, [{
+						mid,
+						seq: seq++,
+						pad: pad(),
+						data: { type: 'text', text, role: 'assistant' }
+					}]);
+					lastBatch = new Date().getTime();
+				}
+			}) : undefined
+		;
+
+		console.log(
+			'continue conversation',
+			options.systemPromptOverride,
+			(options as any).prompt
+		);
+
+		const tools = options.tools ?? this.tools;
+
+		const result = await this.llm.continueConversation({
+			systemPrompt: options.systemPromptOverride,
+			history: 'history' in options ? options.history : [{
+				role: 'user',
+				content: options.prompt
+			}],
+			onChunk,
+			...(models.length > 0 ? { models } : {}),
+			...(tools ? { tools } : {}),
+		});
+
+		
+		if (batch.length > 0) {
+			const text = batch.join('');
+			await this.realtime.publish(options.conversationId, [{
+				mid,
+				seq: seq++,
+				pad: pad(),
+				data: { type: 'text', text, role: 'assistant' }
+			}]);
+		}
+
+		return this.addMessage(options.conversationId, mid, result);
+	}
+
+	async createConversation(user: User): Promise<Conversation> {
+		const createdAt = Date.now();
+		const timestamp = new Date().toLocaleString();
+		const name = `Conversation ${timestamp}`;
+		const userId = user.id;
+
+		for (let i = 0; i < 10; i++) {
+			const conversationId = randomUUID();
+			const conversation = {
+				conversationId, userId, createdAt, name
+			} satisfies Conversation;
+			await this.conversations.save(conversation, { onlyIfNotExists: true });
+			return conversation;
+		}
+
+		throw new Error("Could not create a unique conversation ID!");
+	}
+
+	async updateConversationName(conversationId: string, name: string): Promise<void> {
+		const conversation = await this.getConversation(conversationId);
+		if (!conversation) throw new Error("Conversation doesn't exist.");
+		conversation.name = name;
+		await this.conversations.save(conversation);
+		await this.sendControlMessage(conversationId, {
+			type: 'title',
+			value: name
+		});
+	}
+
+	async getConversation(conversationId: string): Promise<Conversation | undefined> {
+		return this.conversations.get({ conversationId });
+	}
+
+	async listUserConversations(user: User): Promise<Conversation[]> {
+		const conversationsGen = this.conversations.query({
+			by: 'userId-createdAt',
+			where: { userId: { eq: user.id } }
+		});
+		const conversations: Conversation[] = await fromAsync(conversationsGen);
+		return conversations.sort((a, b) => b.createdAt - a.createdAt);
+	}
+
+	async assertUserIsAuthorized(
+		user: User,
+		conversation: string | Conversation | undefined
+	) : Promise<void> {
+		if (typeof conversation === 'string') {
+			const record = await this.getConversation(conversation);
+			return this.assertUserIsAuthorized(user, record);
+		} else if (conversation?.userId !== user.id) {
+			throw new Error("Not authorized");
+		}
+	}
+
+	async deleteConversation(conversationId: string): Promise<void> {
+		// start with the header. if needs be, individual messages can be cleaned up
+		// later. whereas if we start with the messages and are interrupted, we'd just
+		// but corrupting a conversation.
+		await this.conversations.delete({ conversationId });
+
+		const messagesGen = this.messages.query({
+			by: 'conversationId-mid',
+			where: { conversationId: { eq: conversationId } }
+		});
+		const messagesToDelete = await fromAsync(messagesGen);
+		await Promise.all(messagesToDelete.map(msg => this.messages.delete(msg)));
+	}
+
+	async getHistory(conversationId: string): Promise<ConversationMessage[]> {
+		const storedMessages = this.messages.query({
+			by: 'conversationId-mid',
+			where: { conversationId: { eq: conversationId } }
+		});
+
+		// Convert async generator to array and sort by mid
+		const messagesArray = await fromAsync(storedMessages);
+		messagesArray.sort((a, b) => a.mid - b.mid);
+		return messagesArray;
+	};
+
+	async getMessage(
+		conversationId: string,
+		mid: number
+	): Promise<ConversationMessage | undefined> {
+		return this.messages.get({ conversationId, mid });
+	}
+
+	async addMessage<T extends LLMMessage | WorkflowStep>(
+		conversationId: string,
+		mid: number,
+		message: T,
+		broadcast: boolean = false
+	): Promise<ConversationMessage & T> {
+		const fullMessage: ConversationMessage = {
+			conversationId,
+			mid,
+			...message,
+			createdAt: Date.now()
+		};
+
+		await this.messages.save(fullMessage, { onlyIfNotExists: true });
+
+		if (broadcast) {
+			await this.realtime.publish(conversationId, [{
+				mid,
+				seq: 0,
+				pad: pad(),
+				data: {
+					type: 'text',
+					text: message.content,
+					role: message.role
+				}
+			}]);
+		}
+
+		return fullMessage;
+	};
+
+	getStream(context: Context, conversationId: string) {
+		return this.realtime.getStream(context, conversationId);
+	}
+
+	async sendControlMessage(conversationId: string, data: ChunkData, mid: number = -1): Promise<void> {
+		await this.realtime.publish(conversationId, [{
+			mid,
+			seq: 0,
+			pad: pad(),
+			data
+		}]);
+	}
+}
+
+const makeRealtimeService = (scope: Resource) => new RealtimeService<Chunk>(scope, 'realtime');
+
+const makeConversationsTable = (scope: Resource) => new DistributedTable(
+	scope,
+	'conversations',
+	{
+		parse: PassThruParser<Conversation>,
+		key: {
+			partition: { field: 'conversationId', type: 'string' },
+		},
+		indexes: [
+			{
+				partition: { field: 'userId', type: 'string' },
+				sort: { field: 'createdAt', type: 'number' }
+			}
+		],
+	}
+);
+
+const makeMessagesTable = (scope: Resource) => new DistributedTable(
+	scope,
+	'messages',
+	{
+		parse: PassThruParser<ConversationMessage>,
+		key: {
+			partition: { field: 'conversationId', type: 'string' },
+			sort: { field: 'mid', type: 'number' }
+		}
+	}
+);
+
+const makeLLMService = (scope: Resource, systemPrompt?: string, models?: string[]) => 
+	new LLMService(scope, 'llm', {
+		models: models ?? DEFAULT_MODELS_LIST,
+		systemPrompt
+	});
+
+const makeModelsOverrideSetting = (scope: Resource, models?: string[]) => new Setting(scope, 'models', {
+	private: false,
+	init: () => (models ?? DEFAULT_MODELS_LIST).join(', ')
+});

package/templates/default/api/apps/llm/prompts.ts

@@ -0,0 +1,21 @@
+import { dedent } from "./utils.js";
+
+export const generateConversationTitle = (message: string) => dedent`
+	You generate short, descriptive conversation titles based on the user's initial message.
+
+	Rules:
+		- Return ONLY the title text, nothing else
+		- 3-6 words maximum
+		- Capture the main topic or question
+		- No quotes, no explanations
+
+	Examples:
+		- User asks about weather -> "Weather Information Request"
+		- User asks to explain quantum physics -> "Quantum Physics Explanation"  
+		- User asks for recipe help -> "Recipe Assistance"
+		- User asks about programming -> "Programming Question"
+
+	Here is the message:
+		
+	${message}
+`;

package/templates/default/api/apps/llm/tooled-handler.ts

@@ -0,0 +1,116 @@
+import type { ToolCall } from 'wirejs-resources';
+import { Infra } from './infra.js'
+import { cleanTitle } from './utils.js';
+import { generateConversationTitle } from './prompts.js';
+import { standard } from './tools.js';
+
+const assignConversationName = async (infra: Infra, conversationId: string, message: string) => {
+	const titleResponse = await infra.prompt({
+		prompt: generateConversationTitle(message),
+	});
+	const name = cleanTitle(titleResponse.content);
+	console.log('name assigned', name, titleResponse);
+	await infra.updateConversationName(conversationId, name);
+}
+
+export const tooledHandler = (infra: Infra) => async (
+	room: string,
+	newUserMessage: string,
+) => {
+	try {
+		// we can let the user know we're doing stuff immediately.
+		// respectful clients should lock the UI.
+		// TODO: log status events so that newly connecting clients
+		//   also know when the UI should be locked/unlocked.
+		await infra.sendControlMessage(room, { type: 'start' });
+
+		// now, we can get and save state
+		const history = await infra.getHistory(room);
+		let mid = history.length;
+		history.push(await infra.addMessage(room, mid++, {
+			role: 'user', content: newUserMessage
+		}));
+
+		// just getting started and need a conversation title.
+		if (mid === 1) {
+			await assignConversationName(infra, room, newUserMessage);
+		}
+
+		let maxLoops = 10;
+		let toolCalls: ToolCall[];
+		let responseMid = 0;
+		do {
+			const tools = maxLoops > 0 ? standard : undefined;
+			
+			// lets connected clients know we're working on a response now.
+			await infra.sendControlMessage(room, {
+				type: 'status',
+				status: `📝 Responding ...`
+			});
+
+			const response = await infra.respond({
+				conversationId: room,
+				history: history.filter(h => h.role !== 'step'),
+				tools,
+				mid: mid++
+			});
+			responseMid = response.mid;
+			history.push(response);
+
+			toolCalls = tools && tools.length > 0 ? response.tool_calls ?? [] : [];
+			for (const call of toolCalls) {
+				const name = call.function.name;
+				const args = call.function.arguments;
+				const callString = `${name}(${JSON.stringify(args)})`;
+
+				// lets connected clients know we're doing a thing
+				infra.sendControlMessage(room, {
+					type: 'status',
+					status: `⚒️ Calling ${callString}`.slice(0, 100) + '...'
+				}, mid);
+				
+				try {
+					const t = tools!.find(t => t.name === name);	
+					if (!t) throw new Error(`${name} does not exist.`);
+					const r = await t.execute(args);
+
+					// let's future and connected clients know we did a thing
+					history.push(await infra.addMessage(room, mid++, {
+						role: 'step',
+						content: `⚒️ Called ${callString}`.slice(0, 100) + '...',
+					}, true));
+
+					// makes tool results visible in the conversation history
+					// to the agent.
+					history.push(await infra.addMessage(room, mid++, {
+						role: 'tool',
+						tool_name: name,
+						tool_call_id: call.id || JSON.stringify([name, args]),
+						content: JSON.stringify(r, null, 2),
+					}));
+				} catch (error) {
+					history.push(await infra.addMessage(room, mid++, {
+						role: 'tool',
+						tool_name: name,
+						tool_call_id: call.id || JSON.stringify([name, args]),
+						content: String(error),
+					}))
+				}
+			}
+
+			maxLoops--;
+		} while (toolCalls.length > 0);
+
+		// tell connected clients we're done responding (unlock the UI).
+		await infra.sendControlMessage(room, { type: 'end' }, responseMid);
+		
+	} catch (error) {
+		console.error('=== LLM Error ===');
+		console.error('LLM call failed:', error);
+
+		await infra.sendControlMessage(room, {
+			type: 'status',
+			status: 'Internal Error. Please try again.'
+		});
+	}
+};

package/templates/default/api/apps/llm/tools.ts

@@ -0,0 +1,150 @@
+import { dedent, extractContentFromHtml } from "./utils.js";
+import type { ToolDefinition } from "./types.js";
+import { JSDOM } from "jsdom";
+
+const rawFetch = async (url: string) => {
+	try {
+		const parsedUrl = new URL(
+			['http://', 'https://'].some(p => url.startsWith(p)) ? url : `https://${url}`
+		);
+
+		// Use the same fetch logic as httpGet but optimized for analysis
+		const controller = new AbortController();
+		const timeoutId = setTimeout(() => {
+			console.log(`Fetch timeout reached for: ${url}`);
+			controller.abort();
+		}, 20000); // 20 second timeout for analysis
+
+		console.log(`Fetching content from: ${url}`);
+		const request = await fetch(parsedUrl, {
+			signal: controller.signal,
+			headers: {
+				'User-Agent': 'Mozilla/5.0 (compatible; WireJS-Analyzer/1.0)',
+				'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
+			}
+		});
+
+		clearTimeout(timeoutId);
+
+		if (!request.ok) {
+			throw new Error(`HTTP ${request.status}: ${request.statusText}`);
+		}
+
+		const body = await request.text();
+		console.log(`Fetched ${body.length} characters from: ${url}`);
+		
+		return body;
+
+	} catch (error) {
+		console.error(`Error fetching ${url}:`, error);
+		if (error instanceof Error && error.name === 'AbortError') {
+			throw new Error(`Fetch timeout after 20 seconds for: ${url}`);
+		}
+		throw error;
+	}
+}
+
+/**
+ * Standard tool definitions include searching and fetching web content.
+ */
+export const standard: ToolDefinition[] = [
+	{
+		name: 'fetch',
+		description: dedent`
+			Fetches raw content from an HTTP(S) URL via a GET request.
+
+			Use fetch ONLY when the URL is known and expected to contain
+			"raw data" like CSV, JSON, XML, YML, or plain text.
+		`,
+		parameters: {
+			type: 'object',
+			properties: {	
+				url: {
+					type: 'string',
+					description: "Fully qualified URL string to fetch."
+				}
+			}
+		},
+		async execute({ url } : { url: string }) {
+			console.log(`[fetch] Received request for: ${url}`);
+			return rawFetch(url);			
+		}
+	},
+	{
+		name: 'fetch_html_content_text',
+		description: dedent`
+			Extracts the text content from HTML at the given URL.
+
+			Use fetch_html_content_text ONLY when the URL is known and expected
+			to be an HTML page with meaningful content to extract.
+		`,
+		parameters: {
+			type: 'object',
+			properties: {
+				url: {
+					type: 'string',
+					description: "Fully qualified URL string to fetch."
+				}
+			}
+		},
+		async execute({ url } : { url: string }) {
+			console.log(`[natural language fetch] Received request for: ${url}`);
+			const content = extractContentFromHtml(await rawFetch(url));
+			console.log(`Extracted content:\n${content}`);
+			return content;
+		}
+	},
+	{
+		name: 'web_search',
+		description: dedent`
+			Searches the web using DuckDuckGo.
+
+			Use web_search ONLY when user intent indicates a need for information beyond
+			your training or if the subject matter is temporally sensitive AND when a
+			web search is likely to provide the URL you need for a future fetch or
+			fetch_html_content_text call.
+		`,
+		parameters: {
+			type: 'object',
+			properties: {
+				query: {
+					type: 'string',
+					description: 'Search text to use for searching the web. Supports DuckDuckGo search syntax.'
+				}
+			}
+		},
+		async execute({ query } : { query: string }) {
+			console.log(`[searching] query: ${query}`);
+			const rawHtml = await rawFetch(`https://html.duckduckgo.com/html/?q=${query}`);
+			return parseDuckDuckGoResults(rawHtml);
+		}
+	}
+];
+
+const parseDuckDuckGoResults = (html: string) : string => {
+	const results: Array<{ url: string; title: string; description: string }> = [];
+	const dom = new JSDOM(html);
+	const doc = dom.window.document;
+	
+	const resultElements = doc.querySelectorAll('.result');
+	
+	resultElements.forEach((element) => {
+		const titleEl = element.querySelector('.result__a');
+		const snippetEl = element.querySelector('.result__snippet');
+		const linkEl = element.querySelector('a.result__a') as HTMLAnchorElement;
+		
+		if (titleEl && snippetEl && linkEl) {
+			const url = new URL(linkEl.getAttribute('href') || '', 'https://html.duckduckgo.com');
+			const uddgParam = url.searchParams.get('uddg');
+			const decodedUrl = uddgParam ? decodeURIComponent(uddgParam) : '';
+			
+			results.push({
+				url: decodedUrl,
+				title: titleEl.textContent?.trim() || '',
+				description: snippetEl.textContent?.trim() || ''
+			});
+		}
+	});
+	
+	return JSON.stringify(results, null, 2);
+};