create-wirejs-app 2.0.168-llm → 2.0.170

package/templates/default/api/apps/llm/types.ts

@@ -0,0 +1,42 @@
+import { LLMMessage, ToolDefinition as BaseToolDefinition } from "wirejs-resources";
+
+export type Role = 'assistant' | 'user' | 'step' | 'tool';
+
+export type Chunk = {
+	mid: number;
+	seq: number;
+	pad: string; // security padding
+	data: ChunkData;
+}
+
+export type ChunkData =
+	| { type: 'start' }
+	| { type: 'end' }
+	| { type: 'title', value: string }
+	| { type: 'status', status: string }
+	| { type: 'text', text: string, role: Role }
+;
+
+export type Conversation = {
+	userId: string;
+	conversationId: string;
+	name: string;
+	createdAt: number;
+};
+
+export type WorkflowStep = {
+	role: 'step';
+	content: string;
+}
+
+export type ConversationMessage = (WorkflowStep | LLMMessage) & {
+	conversationId: string;
+	mid: number;
+	createdAt: number;
+};
+
+export type ToolDefinition = BaseToolDefinition & {
+	execute: (...args: any) => Promise<any>
+};
+
+export type Message = LLMMessage;

package/templates/default/api/apps/llm/utils.ts

@@ -0,0 +1,164 @@
+
+import { randomUUID } from 'crypto';
+import { JSDOM } from 'jsdom';
+
+/**
+ * Generate a random number of random characters.
+ * 
+ * Intended for inclusion in chunks sent over the wire for security purposes. Chunks
+ * are otherwise more susceptible to snooping.
+ * 
+ * @returns 
+ */
+export const pad = () => randomUUID().slice(0, 1 + Math.floor(Math.random() * 16));
+
+/**
+ * Generate a 64 bit integer ID based on time and randomness. Suitable for
+ * non-globally unique IDs (sub-IDs) that just need to be sortable based on
+ * create or update time.
+ * 
+ * @returns
+ */
+export const intId = () => Math.floor((Date.now() + Math.random()) * 10_000);
+
+/**
+ * Cleans up quotes from a title that may have been provided by an LLM.
+ * 
+ * @param title 
+ * @returns 
+ */
+export const cleanTitle = (title: string) => {
+	let cleanTitle = title.trim();
+	if ((cleanTitle.startsWith('"') && cleanTitle.endsWith('"')) ||
+		(cleanTitle.startsWith("'") && cleanTitle.endsWith("'"))) {
+		cleanTitle = cleanTitle.slice(1, -1).trim();
+	}
+	return cleanTitle;
+}
+
+/**
+ * Extract "content"-like text from HTML using common patterns to ignore headers, footers,
+ * navigation, extra whitespace, etc.
+ */
+export const extractContentFromHtml = (html: string): string => {
+	if (!html.includes('<html') && !html.includes('<!DOCTYPE')) {
+		return html;
+	}
+
+	try {
+		console.log(`[HTML] Starting JSDOM extraction from ${html.length} chars`);
+
+		// Load HTML into JSDOM for DOM manipulation
+		const dom = new JSDOM(html);
+		const document = dom.window.document;
+
+		// Remove unwanted elements entirely using querySelectorAll and remove
+		const removeSelectors = [
+			'script, style, nav, header, footer, aside',
+			'.mw-navigation, .navbox, .infobox, .sidebar', // Wikipedia-specific
+			'[class*="nav"], [class*="menu"], [class*="sidebar"]', // Common patterns
+			'.reference, .citation, sup.reference', // Citations
+			'.printfooter, .catlinks', // Wikipedia footer stuff
+			'table.ambox, .hatnote' // Wikipedia message boxes
+		];
+
+		removeSelectors.forEach(selector => {
+			document.querySelectorAll(selector).forEach(element => element.remove());
+		});
+
+		// Extract text with cleaned up whitespace
+		const bodyElement = document.body;
+		let text = (bodyElement?.textContent || '')
+			.replace(/\s+/g, ' ')          // Normalize whitespace
+			.replace(/\[\d+\]/g, '')       // Remove citation numbers [1], [2], etc.
+			.replace(/\s*\n\s*/g, '\n')    // Clean line breaks
+			.replace(/\n{3,}/g, '\n\n')    // Limit consecutive newlines
+			.trim();
+
+		console.log(`[HTML] Final JSDOM extracted text: ${text.length} chars`);
+		return text;
+	} catch (error) {
+		console.error('Error extracting text with JSDOM:', error);
+		// Fallback to simple regex approach
+		console.log('[HTML] Falling back to regex extraction');
+		return html.replace(/<[^>]*>/g, ' ').replace(/\s+/g, ' ').trim();
+	}
+};
+
+/**
+ * Convert async generator to array (for Node 20 compatibility)
+ */
+export async function fromAsync<T>(gen: AsyncGenerator<T>): Promise<T[]> {
+	const items: T[] = [];
+	for await (const item of gen) {
+		items.push(item);
+	}
+	return items;
+}
+
+/**
+ * Removes leading whitespace on every line of input text.
+ * 
+ * This works by finding the minimum indent depth of a block of multi-line text and
+ * left-trimming every line by the common (minimum) indent amount.
+ * 
+ * @param content 
+ * @returns 
+ */
+function dedentString(content: string) {
+	return dedentLines(content.split('\n')).join('\n');
+}
+
+function findMinimumIndent(lines: string[]) {
+	let minimumIndent: number | undefined = undefined;
+	for (const line of lines) {
+		if (line.trim() === '') continue;
+
+		const whitespace = line.match(/^\s+/)?.[0];
+		if (!whitespace) continue;
+
+		if (minimumIndent === undefined || whitespace.length < minimumIndent) {
+			minimumIndent = whitespace.length;
+		}
+	}
+	return minimumIndent;
+}
+
+function dedentLines(lines: string[]) {
+	const output: string[] = [];
+	const minimumIndent = findMinimumIndent(lines);
+	for (const line of lines) {
+		output.push(line.substring(minimumIndent ?? 0));
+	}
+	return output;
+}
+
+export function dedent(content: TemplateStringsArray, ...values: any[]) {
+	const PH = '%%%___DEDENT_PLACEHOLDER___%%%';
+	const combined = Array.from(content).join(PH);
+	const dedentedContent = dedentString(combined).split(PH);
+	return String.raw({ raw: dedentedContent }, ...values);
+}
+
+export function parseLLMJson(raw: string) {
+	if (!raw || typeof raw !== "string") {
+		throw new Error("Empty or non-string LLM output");
+	}
+
+	let text = raw.trim();
+
+	if (text.startsWith("```")) {
+		text = text.replace(/^```[a-zA-Z]*\s*/, "");
+		text = text.replace(/```$/, "");
+		text = text.trim();
+	}
+
+	if (text.toLowerCase().startsWith("json")) {
+		const after = text.slice(4).trim();
+		if (after.startsWith("{")) {
+			text = after;
+		}
+	}
+
+	return JSON.parse(text);
+}

package/templates/default/api/index.ts

@@ -4,12 +4,12 @@ import { Todos } from './apps/todos.js';
 import { Wiki } from './apps/wiki.js';
 import { Store } from './apps/store.js';
 import { Admin } from './apps/admin.js';
-import { LLM } from './apps/llm.js';
+import { LLM } from './apps/llm/index.js';
 
+export type * from './apps/llm/index.js';
 export type * from './apps/todos.js';
 export type * from './apps/store.js';
 export type * from './apps/admin.js';
-export type * from './apps/llm.js';
 
 const authService = new AuthenticationService('app', 'core-users');
 
@@ -22,16 +22,18 @@ export const admin = Admin(auth);
 export const llm = LLM(auth);
 
 new Endpoint('app', 'sample-endpoint', {
-	description: "Sample endpoint to show dynamic endpoint creation.",
-	handle() {
+	description: "Sample endpoint to show programmatic endpoint creation.",
+	handle(context) {
+		context.responseHeaders['Content-Type'] = 'text/html; charset=utf-8';
 		return "<html><body><p>Hello!</p><p><a href='/'>Back.</a></body></html>";
 	}
 });
 
 new Endpoint('app', 'sample-wildcard-endpoint', {
 	path: 'wildcard-endpoint/%',
-	description: "Sample endpoint to show dynamic wildcard endpoint creation.",
+	description: "Sample endpoint to show programmatic wildcard endpoint creation.",
 	handle(context) {
+		context.responseHeaders['Content-Type'] = 'text/html; charset=utf-8';
 		return `<html>
 			<body>
 				<h2>${context.location.toString()
@@ -42,4 +44,4 @@ new Endpoint('app', 'sample-wildcard-endpoint', {
 			</body>
 		</html>`;
 	}
-});
+});

package/templates/default/api/package.json

@@ -11,5 +11,13 @@
 		"types": "./index.ts",
 		"wirejs:client": "./dist/index.client.js",
 		"default": "./dist/index.js"
+	},
+	"dependencies": {
+		"jsdom": "^25.0.0",
+		"wirejs-module-payments-stripe": "*",
+		"wirejs-resources": "*"
+	},
+	"devDependencies": {
+		"@types/jsdom": "^21.0.0"
 	}
 }

package/templates/default/deployment-config.ts

@@ -0,0 +1,6 @@
+import { DeploymentConfig } from 'wirejs-resources';
+
+export default {
+	runtimeDesiredMemoryMB: 1 * 1024,
+	bundleNodeModules: ['jsdom'],
+} satisfies DeploymentConfig;

package/templates/default/package.json

@@ -12,13 +12,13 @@
 		"dompurify": "^3.2.3",
 		"marked": "^15.0.6",
 		"wirejs-dom": "^1.0.44",
-		"wirejs-resources": "^0.1.163-llm",
-		"wirejs-components": "^0.1.106-llm",
-		"wirejs-module-payments-stripe": "^0.1.57-llm",
-		"wirejs-web-worker": "^1.0.60-llm"
+		"wirejs-resources": "^0.1.165",
+		"wirejs-components": "^0.1.108",
+		"wirejs-module-payments-stripe": "^0.1.59",
+		"wirejs-web-worker": "^1.0.62"
 	},
 	"devDependencies": {
-		"wirejs-scripts": "^3.0.161-llm",
+		"wirejs-scripts": "^3.0.163",
 		"typescript": "^5.7.3"
 	},
 	"scripts": {

package/templates/default/src/ssg/index.ts

@@ -9,11 +9,11 @@ export async function generate() {
 			<p>It comes with some sample API methods and pages.</p>
 			<ul>
 				<li><a href='/todo-app.html'>Todo App</a></li>
-				<li><a href='/simple-wiki/index.html'>Simple Wiki</a></li>
+				<li><a href='/simple-wiki/index'>Simple Wiki</a></li>
 				<li><a href='/realtime-test.html'>Realtime Test</a></li>
 				<li><a href='/web-worker-test.html'>Web Worker Test</a></li>
 				<li><a href='/storefront.html'>Storefront</a></li>
-				<li><a href='/chatbot.html'>Chatbot</a></li>
+				<li><a href='/llm-test.html'>LLM Test</a></li>
 				<li><a href='/admin.html'>Admin</a></li>
 			</ul>
 		</div>`