luca 1.1.2 → 3.0.0

package/src/web/features/vm.ts

@@ -0,0 +1,85 @@
+// @ts-nocheck
+//
+import { z } from 'zod'
+import { FeatureStateSchema, FeatureOptionsSchema } from '../../schemas/base.js'
+import vm from '../shims/isomorphic-vm'
+import { Feature } from "../feature.js";
+
+export const VMStateSchema = FeatureStateSchema.extend({})
+
+export const VMOptionsSchema = FeatureOptionsSchema.extend({
+  context: z.any().describe('VM context object').optional(),
+})
+
+export type VMState = z.infer<typeof VMStateSchema>
+export type VMOptions = z.infer<typeof VMOptionsSchema>
+
+/**
+ * Sandboxed JavaScript execution environment for the browser.
+ *
+ * Automatically injects the container's context object into the global scope,
+ * so evaluated code can use anything provided by the container. Useful for
+ * live code playgrounds, plugin systems, and dynamic script evaluation.
+ *
+ * @extends Feature
+ *
+ * @example
+ * ```typescript
+ * const vm = container.feature('vm')
+ * const result = vm.run('1 + 2 + 3') // 6
+ * const greeting = vm.run('container.uuid') // accesses container globals
+ * ```
+ */
+export class VM<
+  T extends VMState = VMState,
+  K extends VMOptions = VMOptions
+> extends Feature<T, K> {
+
+  static override stateSchema = VMStateSchema
+  static override optionsSchema = VMOptionsSchema
+  static override shortcut = "features.vm" as const
+
+  static { Feature.register(this, 'vm') }
+
+  createScript(code: string) {
+    return new vm.Script(code)
+  }
+ 
+  createContext(ctx: any = {}) {
+    return vm.createContext({
+      ...this.container.context,
+      ...ctx
+    })
+  }
+  
+  async run(code: string, ctx: any = {}, options : any = {}) {
+    let script = this.createScript(code)
+    
+    if (options.transform) {
+      const esbuild = this.container.feature('esbuild')
+      await esbuild.start()
+      const result = await esbuild.compile(code, options.transform)
+      script = this.createScript(result.code)
+    }
+
+    const context = this.createContext(ctx)
+    
+    try {
+      const result = script.runInContext({
+        ...context,
+        ...(options.exports && { exports: options.exports })
+      })
+  
+      if (options.exports) {
+        return { result, exports: options.exports, context }
+      }
+      
+      return result
+    } catch(error) {
+      console.error(`Error running code`, error)
+      return error
+    }
+  }
+}
+
+export default VM;

package/src/web/features/voice-recognition.ts

@@ -0,0 +1,161 @@
+import { z } from 'zod'
+import { FeatureStateSchema, FeatureOptionsSchema, FeatureEventsSchema } from '../../schemas/base.js'
+import { Feature } from "../../feature.js";
+import { type ContainerContext } from '../container.js'
+
+export const VoiceRecognitionOptionsSchema = FeatureOptionsSchema.extend({
+  language: z.string().optional().describe('BCP 47 language code for recognition (e.g. en-US)'),
+  continuous: z.boolean().optional().describe('Whether to continuously listen for speech'),
+  autoListen: z.boolean().optional().describe('Whether to automatically start listening on creation'),
+})
+
+export const VoiceRecognitionStateSchema = FeatureStateSchema.extend({
+  listening: z.boolean().describe('Whether the recognizer is currently listening'),
+  transcript: z.string().describe('Accumulated final transcript text'),
+})
+
+export type VoiceRecognitionOptions = z.infer<typeof VoiceRecognitionOptionsSchema>
+export type VoiceRecognitionState = z.infer<typeof VoiceRecognitionStateSchema>
+
+export const VoiceRecognitionEventsSchema = FeatureEventsSchema.extend({
+  result: z.tuple([
+    z.object({
+      finalTranscript: z.string().describe('Accumulated final transcript text'),
+      interimTranscript: z.string().describe('Current interim transcript text'),
+    }).describe('Recognition result payload'),
+  ]).describe('Fires when speech recognition produces a result'),
+  error: z.tuple([z.string().describe('Error message from the recognizer')]).describe('Fires when speech recognition encounters an error'),
+  end: z.tuple([]).describe('Fires when speech recognition ends'),
+  start: z.tuple([]).describe('Fires when speech recognition starts listening'),
+  stop: z.tuple([]).describe('Fires when speech recognition is manually stopped'),
+  abort: z.tuple([]).describe('Fires when speech recognition is aborted'),
+}).describe('VoiceRecognition events')
+
+/**
+ * Speech-to-text recognition using the Web Speech API (SpeechRecognition).
+ *
+ * Wraps the browser's built-in speech recognition, supporting continuous
+ * listening, interim results, and language selection. Recognized text is
+ * accumulated in state and emitted as events for real-time transcription UIs.
+ *
+ * @extends Feature
+ *
+ * @example
+ * ```typescript
+ * const voice = container.feature('voice', { continuous: true, autoListen: true })
+ *
+ * voice.on('transcript', ({ text }) => {
+ *   console.log('Heard:', text)
+ * })
+ *
+ * // Or start manually
+ * voice.start()
+ * ```
+ */
+export class VoiceRecognition<T extends VoiceRecognitionState = VoiceRecognitionState, K extends VoiceRecognitionOptions = VoiceRecognitionOptions> extends Feature<T, K> {
+  // @ts-ignore-next-line
+  private recognition: SpeechRecognition | null = null;
+
+  static override stateSchema = VoiceRecognitionStateSchema
+  static override optionsSchema = VoiceRecognitionOptionsSchema
+  static override eventsSchema = VoiceRecognitionEventsSchema
+  static override shortcut = "features.voice" as const
+
+  static { Feature.register(this as any, 'voice') }
+
+  constructor(options: K, context: ContainerContext) {
+    super(options, context);
+
+    this.state.set("listening", false);
+    this.state.set("transcript", "");
+
+    if (!('webkitSpeechRecognition' in window)) {
+      throw new Error('Voice recognition is not supported in this browser.');
+    }
+
+    // @ts-ignore-next-line
+    this.recognition = new webkitSpeechRecognition();
+    this.recognition.lang = this.options.language || "en-US";
+    this.recognition.continuous = this.options.continuous || false;
+    this.recognition.interimResults = true;
+
+    this.recognition.onresult = (event: any) => {
+      let interimTranscript = "";
+      for (let i = event.resultIndex; i < event.results.length; i++) {
+        const transcript = event.results[i][0].transcript;
+        if (event.results[i].isFinal) {
+          this.state.set("transcript", this.state.get("transcript") + transcript);
+        } else {
+          interimTranscript += transcript;
+        }
+      }
+      this.emit("result", {
+        finalTranscript: this.state.get("transcript"),
+        interimTranscript
+      });
+    };
+
+    this.recognition.onerror = (event: any) => {
+      this.emit("error", event.error);
+    };
+
+    this.recognition.onend = () => {
+      this.state.set("listening", false);
+      this.emit("end");
+    };
+    
+    if (options?.autoListen) {
+      this.start()
+    }
+  }
+
+  /** Whether the speech recognizer is currently listening for audio input. */
+  get listening() {
+    return !!this.state.get("listening");
+  }
+
+  /** Returns the accumulated final transcript text from recognition results. */
+  get transcript() {
+    return this.state.get("transcript") || '';
+  }
+  
+  async whenFinished() {
+    if(!this.listening) {
+      this.start()
+    }
+
+    return await this.waitFor("end").then(() => this.transcript)  
+  }
+
+  start() {
+    this.state.set('transcript', '')
+
+    if (!this.listening && this.recognition) {
+      this.recognition.start();
+      this.state.set("listening", true);
+      this.emit("start");
+    }
+  }
+
+  stop() {
+    if (this.listening && this.recognition) {
+      this.recognition.stop();
+      this.state.set("listening", false);
+      this.emit("stop");
+    }
+  }
+
+  abort() {
+    if (this.listening && this.recognition) {
+      this.recognition.abort();
+      this.state.set("listening", false);
+      this.emit("abort");
+    }
+  }
+
+  clearTranscript() {
+    this.state.set("transcript", "");
+  }
+}
+
+export default VoiceRecognition;

package/src/web/shims/isomorphic-vm.ts

@@ -0,0 +1,149 @@
+const globals = ['Array', 'Boolean', 'Date', 'Error', 'EvalError', 'Function',
+  'Infinity', 'JSON', 'Math', 'NaN', 'Number', 'Object', 'RangeError',
+  'ReferenceError', 'RegExp', 'String', 'SyntaxError', 'TypeError', 'URIError',
+  'decodeURI', 'decodeURIComponent', 'encodeURI', 'encodeURIComponent', 'escape',
+  'eval', 'isFinite', 'isNaN', 'parseFloat', 'parseInt', 'undefined', 'unescape'];
+
+class Context { 
+  [key: string]: any;
+}
+
+function createContext(context?: any): Context {
+  var copy = new Context();
+
+  if (typeof context === 'object') {
+    Object.keys(context).forEach(key => copy[key] = context[key]);
+  }
+
+  return copy;
+};
+
+function isContext(context: any): context is Context {
+  return context instanceof Context;
+}
+
+function runInContext(code: string, context: Context): any {
+  const iframe = document.createElement('iframe');
+
+  if (!iframe.style) {
+    (iframe as any).style = {};
+  }
+
+  iframe.style.display = 'none';
+
+  document.body.appendChild(iframe);
+
+  const iframeWindow: any = iframe.contentWindow;
+  let iframeEval = iframeWindow.eval;
+
+  if (!iframeEval && iframeWindow.execScript) {
+    iframeWindow.execScript.call(iframeWindow, 'null');
+    iframeEval = iframeWindow.eval;
+  }
+
+  Object.keys(context).forEach(key => {
+    iframeWindow[key] = context[key];
+  });
+
+  Object.keys(globals).forEach(key => {
+    if (context[key]) {
+      iframeWindow[key] = context[key];
+    }
+  });
+
+  const result = iframeEval.call(iframeWindow, code);
+
+  Object.keys(iframeWindow).forEach(key => {
+    if (key in context || Object.keys(iframeWindow).indexOf(key) === -1) {
+      context[key] = iframeWindow[key];
+    }
+  });
+
+  globals.forEach(key => {
+    if (!(key in context)) {
+      Object.defineProperty(context, key, {
+        writable: true,
+        enumerable: false,
+        configurable: true,
+        value: iframeWindow[key],
+      });
+    }
+  });
+
+  document.body.removeChild(iframe);
+
+  return result;
+};
+
+function runInDebugContext(code: string): any {
+  return runInNewContext(code);
+};
+
+function runInNewContext(code: string, context?: any): any {
+  const contextInstance = createContext(context);
+  const result = runInContext(code, contextInstance);
+
+  if (typeof context === 'object') {
+    Object.keys(contextInstance).forEach(key => {
+      context[key] = contextInstance[key];
+    });
+  }
+
+  return result;
+};
+
+function runInThisContext(code: string): any {
+  return (0, eval)(code);
+};
+
+class Script {
+  private code: string;
+  constructor(code: string) {
+    this.code = code;
+  }
+
+  runInContext(context: Context) {
+    return runInContext(this.code, context);
+  }
+
+  runInNewContext(context?: any) {
+    return runInNewContext(this.code, context);
+  }
+
+  runInThisContext() {
+    return runInThisContext(this.code);
+  }
+}
+
+export interface VMContext {
+  [name: string]: any;
+}
+
+export interface VMScript {
+  new (code: string) : VMScript;
+  runInContext(context: VMContext) : any;
+  runInNewContext(context?: any) : any;
+  runInThisContext() : any;
+}
+
+export interface VM {
+  Script: VMScript;
+  createContext(context?: any): VMContext;
+  isContext(context: any): context is VMContext;
+  runInContext(code: string, context: VMContext): any;
+  runInDebugContext(code: string): any;
+  runInNewContext(code: string, context?: any): any;
+  runInThisContext(code: string): any;
+}
+
+export const shim: VM = <any>{
+  Script,
+  createContext,
+  isContext,
+  runInContext,
+  runInDebugContext,
+  runInNewContext,
+  runInThisContext,
+};
+
+export default shim as VM 

package/test/assistant-hooks.test.ts

@@ -0,0 +1,306 @@
+import { describe, it, expect, beforeEach } from 'bun:test'
+import { AGIContainer } from '../src/agi/container.server'
+import type { Assistant } from '../src/agi/features/assistant'
+
+/**
+ * Helper: create a runtime assistant with no folder, stub the conversation
+ * so tests never hit the network, and inject hook functions directly.
+ */
+function makeAssistant(hooks: Record<string, (...args: any[]) => any> = {}) {
+	const container = new AGIContainer()
+	const assistant = container.feature('assistant', {
+		systemPrompt: 'You are a test assistant.',
+		model: 'gpt-5',
+	}) as Assistant
+
+	// Inject hooks into state (same shape loadHooks() produces)
+	assistant.state.set('hooks', hooks)
+
+	return assistant
+}
+
+/** Start + stub so ask() never touches the network */
+async function startWithStub(assistant: Assistant, pattern: string | RegExp = /.*/, response: string = 'stubbed') {
+	await assistant.start()
+	assistant.conversation.stub(pattern, response)
+	return assistant
+}
+
+describe('Assistant triggerHook', () => {
+	it('returns undefined when no hook exists for the name', async () => {
+		const assistant = makeAssistant()
+		const result = await assistant.triggerHook('nonexistent', 'arg1')
+		expect(result).toBeUndefined()
+	})
+
+	it('calls the hook with (assistant, ...args) and awaits it', async () => {
+		const calls: any[][] = []
+		const assistant = makeAssistant({
+			myHook: async (asst: any, a: string, b: number) => {
+				calls.push([asst, a, b])
+				return 'hook-result'
+			},
+		})
+
+		const result = await assistant.triggerHook('myHook', 'hello', 42)
+		expect(result).toBe('hook-result')
+		expect(calls).toHaveLength(1)
+		expect(calls[0]![0]).toBe(assistant)
+		expect(calls[0]![1]).toBe('hello')
+		expect(calls[0]![2]).toBe(42)
+	})
+
+	it('emits hookFired before calling the hook', async () => {
+		const events: string[] = []
+		const assistant = makeAssistant({
+			myHook: async () => { events.push('hook-ran') },
+		})
+		assistant.on('hookFired', (name: string) => { events.push(`hookFired:${name}`) })
+
+		await assistant.triggerHook('myHook')
+		expect(events).toEqual(['hookFired:myHook', 'hook-ran'])
+	})
+
+	it('awaits async hooks to completion before returning', async () => {
+		let resolved = false
+		const assistant = makeAssistant({
+			slowHook: async () => {
+				await new Promise(r => setTimeout(r, 50))
+				resolved = true
+			},
+		})
+
+		await assistant.triggerHook('slowHook')
+		expect(resolved).toBe(true)
+	})
+})
+
+describe('Assistant lifecycle hooks', () => {
+	it('beforeStart runs before the assistant is started', async () => {
+		const order: string[] = []
+		const assistant = makeAssistant({
+			beforeStart: async () => { order.push('beforeStart') },
+			started: async () => { order.push('started') },
+			afterStart: async () => { order.push('afterStart') },
+		})
+
+		await assistant.start()
+		expect(order).toEqual(['beforeStart', 'started', 'afterStart'])
+	})
+
+	it('afterStart blocks start() until complete', async () => {
+		let afterStartDone = false
+		const assistant = makeAssistant({
+			afterStart: async () => {
+				await new Promise(r => setTimeout(r, 50))
+				afterStartDone = true
+			},
+		})
+
+		await assistant.start()
+		expect(afterStartDone).toBe(true)
+	})
+
+	it('formatSystemPrompt modifies the system prompt before conversation creation', async () => {
+		const assistant = makeAssistant({
+			formatSystemPrompt: async (_asst: any, prompt: string) => {
+				return prompt + '\nExtra instructions.'
+			},
+		})
+
+		await assistant.start()
+		expect(assistant.systemPrompt).toContain('Extra instructions.')
+	})
+})
+
+describe('Assistant ask hooks', () => {
+	it('beforeAsk fires on every ask() call', async () => {
+		let callCount = 0
+		const assistant = makeAssistant({
+			beforeAsk: async () => { callCount++ },
+		})
+		await startWithStub(assistant)
+
+		await assistant.ask('first')
+		await assistant.ask('second')
+		expect(callCount).toBe(2)
+	})
+
+	it('beforeAsk can rewrite the question via return value', async () => {
+		const questionsReceived: string[] = []
+		const assistant = makeAssistant({
+			beforeAsk: async (_asst: any, question: string) => {
+				return question + ' (rewritten)'
+			},
+		})
+		await assistant.start()
+
+		// Stub that captures the actual question sent to the conversation
+		assistant.conversation.on('userMessage', (content: any) => {
+			questionsReceived.push(typeof content === 'string' ? content : JSON.stringify(content))
+		})
+		assistant.conversation.stub(/.*/, 'ok')
+
+		await assistant.ask('hello')
+		expect(questionsReceived[0]).toContain('(rewritten)')
+	})
+
+	it('beforeInitialAsk fires only on the first ask()', async () => {
+		let callCount = 0
+		const assistant = makeAssistant({
+			beforeInitialAsk: async () => { callCount++ },
+		})
+		await startWithStub(assistant)
+
+		await assistant.ask('first')
+		await assistant.ask('second')
+		await assistant.ask('third')
+		expect(callCount).toBe(1)
+	})
+
+	it('answered hook fires after the response is produced', async () => {
+		let hookResponse: string | undefined
+		const assistant = makeAssistant({
+			answered: async (_asst: any, result: string) => {
+				hookResponse = result
+			},
+		})
+		await startWithStub(assistant, /.*/, 'the answer')
+
+		await assistant.ask('question')
+		expect(hookResponse).toBe('the answer')
+	})
+
+	it('answered hook is awaited before ask() returns', async () => {
+		let hookDone = false
+		const assistant = makeAssistant({
+			answered: async () => {
+				await new Promise(r => setTimeout(r, 50))
+				hookDone = true
+			},
+		})
+		await startWithStub(assistant)
+
+		await assistant.ask('question')
+		expect(hookDone).toBe(true)
+	})
+})
+
+describe('Assistant tool hooks', () => {
+	it('beforeToolCall hook fires before tool execution', async () => {
+		const hookCalls: any[] = []
+		const assistant = makeAssistant({
+			beforeToolCall: async (_asst: any, ctx: any) => {
+				hookCalls.push({ name: ctx.name, args: { ...ctx.args } })
+			},
+		})
+		await assistant.start()
+
+		// Add a simple tool and stub a response that triggers it
+		assistant.addTool('echo', {
+			description: 'Echo tool',
+			parameters: { type: 'object', properties: { text: { type: 'string' } }, required: ['text'] },
+			handler: async (args: any) => args.text,
+		})
+
+		// Directly invoke the toolExecutor to test hooks without needing model responses
+		const handler = async (args: any) => args.text
+		await assistant.conversation.toolExecutor!('echo', { text: 'hello' }, handler)
+
+		expect(hookCalls).toHaveLength(1)
+		expect(hookCalls[0]!.name).toBe('echo')
+		expect(hookCalls[0]!.args.text).toBe('hello')
+	})
+
+	it('afterToolCall hook fires after tool execution with result', async () => {
+		const hookCalls: any[] = []
+		const assistant = makeAssistant({
+			afterToolCall: async (_asst: any, ctx: any) => {
+				hookCalls.push({ name: ctx.name, result: ctx.result })
+			},
+		})
+		await assistant.start()
+
+		const handler = async (args: any) => args.text
+		await assistant.conversation.toolExecutor!('echo', { text: 'world' }, handler)
+
+		expect(hookCalls).toHaveLength(1)
+		expect(hookCalls[0]!.name).toBe('echo')
+		expect(hookCalls[0]!.result).toBe('world')
+	})
+
+	it('beforeToolCall can modify args via ctx mutation', async () => {
+		const assistant = makeAssistant({
+			beforeToolCall: async (_asst: any, ctx: any) => {
+				ctx.args.text = 'intercepted'
+			},
+		})
+		await assistant.start()
+
+		let receivedArgs: any
+		const handler = async (args: any) => { receivedArgs = args; return 'ok' }
+		await assistant.conversation.toolExecutor!('echo', { text: 'original' }, handler)
+
+		expect(receivedArgs.text).toBe('intercepted')
+	})
+
+	it('beforeToolCall can skip execution via ctx.skip', async () => {
+		const assistant = makeAssistant({
+			beforeToolCall: async (_asst: any, ctx: any) => {
+				ctx.skip = true
+				ctx.result = 'skipped-by-hook'
+			},
+		})
+		await assistant.start()
+
+		let handlerCalled = false
+		const handler = async () => { handlerCalled = true; return 'should not run' }
+		const result = await assistant.conversation.toolExecutor!('echo', { text: 'hi' }, handler)
+
+		expect(handlerCalled).toBe(false)
+		expect(result).toBe('skipped-by-hook')
+	})
+
+	it('afterToolCall can modify the result via ctx mutation', async () => {
+		const assistant = makeAssistant({
+			afterToolCall: async (_asst: any, ctx: any) => {
+				ctx.result = 'modified-result'
+			},
+		})
+		await assistant.start()
+
+		const handler = async () => 'original-result'
+		const result = await assistant.conversation.toolExecutor!('echo', {}, handler)
+
+		expect(result).toBe('modified-result')
+	})
+})
+
+describe('Assistant forwarded event hooks', () => {
+	it('turnStart hook fires before the turnStart event on the assistant bus', async () => {
+		const order: string[] = []
+		const assistant = makeAssistant({
+			turnStart: async () => { order.push('hook') },
+		})
+		await startWithStub(assistant)
+
+		assistant.on('turnStart', () => { order.push('event') })
+
+		await assistant.ask('go')
+		expect(order[0]).toBe('hook')
+		expect(order[1]).toBe('event')
+	})
+
+	it('response hook fires with the final response text', async () => {
+		let hookText: string | undefined
+		const assistant = makeAssistant({
+			response: async (_asst: any, text: string) => {
+				hookText = text
+			},
+		})
+		await startWithStub(assistant, /.*/, 'final answer')
+
+		await assistant.ask('question')
+		expect(hookText).toBe('final answer')
+	})
+})