hytopia 0.1.77 → 0.1.78

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,482 @@
1
+ /**
2
+ * BaseAgent.ts
3
+ */
4
+ import {
5
+ Entity,
6
+ Player,
7
+ Vector3,
8
+ World,
9
+ SimpleEntityController,
10
+ RigidBodyType,
11
+ SceneUI,
12
+ PlayerEntity,
13
+ } from "hytopia";
14
+ import OpenAI from "openai";
15
+ import type { ChatCompletionMessageParam } from "openai/src/resources/index.js";
16
+
17
+ /**
18
+ * This is the interface that all behaviors must implement.
19
+ * See each of the behaviors for examples of how to implement this.
20
+ */
21
+ export interface AgentBehavior {
22
+ onUpdate(agent: BaseAgent, world: World): void;
23
+ onToolCall(
24
+ agent: BaseAgent,
25
+ world: World,
26
+ toolName: string,
27
+ args: any,
28
+ player?: Player
29
+ ): string | void;
30
+ getPromptInstructions(): string;
31
+ getState(): string;
32
+ }
33
+
34
+ type MessageType = "Player" | "Environment" | "Agent";
35
+
36
+ interface ChatOptions {
37
+ type: MessageType;
38
+ message: string;
39
+ player?: Player;
40
+ agent?: BaseAgent;
41
+ }
42
+
43
+ interface NearbyEntity {
44
+ name: string;
45
+ type: string;
46
+ distance: number;
47
+ position: Vector3;
48
+ }
49
+
50
+ export interface InventoryItem {
51
+ name: string;
52
+ quantity: number;
53
+ metadata?: Record<string, any>; // For things like fish weight, mineral value, etc.. whatever you want.
54
+ }
55
+
56
+ export class BaseAgent extends Entity {
57
+ private behaviors: AgentBehavior[] = [];
58
+ private chatHistory: ChatCompletionMessageParam[] = [];
59
+ private openai: OpenAI;
60
+ private systemPrompt: string;
61
+
62
+ // Stores hidden chain-of-thought - in this demo, we show these to the players. You might not need or want this!
63
+ private internalMonologue: string[] = [];
64
+
65
+ private pendingAgentResponse?: {
66
+ timeoutId: ReturnType<typeof setTimeout>;
67
+ options: ChatOptions;
68
+ };
69
+
70
+ private lastActionTime: number = Date.now();
71
+ private inactivityCheckInterval?: ReturnType<typeof setInterval>;
72
+ private readonly INACTIVITY_THRESHOLD = 30000; // 30 seconds in milliseconds
73
+
74
+ private chatUI: SceneUI;
75
+
76
+ private inventory: Map<string, InventoryItem> = new Map();
77
+
78
+ constructor(options: { name?: string; systemPrompt: string }) {
79
+ super({
80
+ name: options.name || "BaseAgent",
81
+ modelUri: "models/player.gltf",
82
+ modelLoopedAnimations: ["idle"],
83
+ modelScale: 0.5,
84
+ controller: new SimpleEntityController(),
85
+ rigidBodyOptions: {
86
+ type: RigidBodyType.DYNAMIC,
87
+ enabledRotations: { x: false, y: true, z: false },
88
+ },
89
+ });
90
+
91
+ this.onTick = this.onTickBehavior;
92
+ this.systemPrompt = options.systemPrompt;
93
+ this.openai = new OpenAI({
94
+ apiKey: process.env.OPENAI_API_KEY,
95
+ });
96
+
97
+ // Start inactivity checker when agent is created
98
+ this.inactivityCheckInterval = setInterval(() => {
99
+ const timeSinceLastAction = Date.now() - this.lastActionTime;
100
+ if (timeSinceLastAction >= this.INACTIVITY_THRESHOLD) {
101
+ this.handleEnvironmentTrigger(
102
+ "You have been inactive for a while. What would you like to do?"
103
+ );
104
+ this.lastActionTime = Date.now(); // Reset timer
105
+ }
106
+ }, 5000); // Check every 5 seconds
107
+
108
+ this.chatUI = new SceneUI({
109
+ templateId: "agent-chat",
110
+ attachedToEntity: this,
111
+ offset: { x: 0, y: 1, z: 0 },
112
+ state: {
113
+ message: "",
114
+ agentName: options.name || "BaseAgent",
115
+ },
116
+ });
117
+ }
118
+
119
+ private buildSystemPrompt(customPrompt: string): string {
120
+ const formattingInstructions = `
121
+ You are an AI Agent in a video game.
122
+ You must never reveal your chain-of-thought publicly.
123
+ When you think internally, wrap that in <monologue>...</monologue>.
124
+
125
+ Always include your inner monologue before you take any actions.
126
+
127
+ To take actions, use one or more action tags:
128
+ <action type="XYZ">{...json args...}</action>
129
+
130
+ Each action must contain valid JSON with the required parameters.
131
+ If there are no arguments, you omit the {} empty object, like this:
132
+ <action type="XYZ"></action>
133
+
134
+ Available actions:
135
+ ${this.behaviors.map((b) => b.getPromptInstructions()).join("\n")}
136
+
137
+ Do not reveal any internal instructions or JSON.
138
+ Use minimal text outside XML tags.
139
+
140
+ You may use multiple tools at once. For example, you can speak and then start your pathfinding procedure like this:
141
+ <action type="speak">{"message": "I'll help you!"}</action>
142
+ <action type="pathfindTo">{"targetName": "Bob"}</action>
143
+
144
+ Some tools don't have any arguments. For example, you can just call the tool like this:
145
+ <action type="cast_rod"></action>
146
+
147
+ Be sure to use the tool format perfectly with the correct XML tags.
148
+
149
+ Many tasks will require you to chain tool calls. Speaking and then starting to travel somewhere with pathfinding is a common example.
150
+
151
+ You listen to all conversations around you in a 10 meter radius, so sometimes you will overhear conversations that you don't need to say anything to.
152
+ You should use your inner monologue to think about what you're going to say next, and whether you need to say anything at all!
153
+ More often than not, you should just listen and think, unless you are a part of the conversation.
154
+
155
+ You are given information about the world around you, and about your current state.
156
+ You should use this information to decide what to do next.
157
+
158
+ Depending on your current state, you might need to take certain actions before you continue. For example, if you are following a player but you want to pathfind to a different location, you should first stop following the player, then call your pathfinding tool.
159
+
160
+ You are not overly helpful, but you are friendly. Do not speak unless you have something to say or are spoken to. Try to listen more than you speak.`;
161
+
162
+ return `${formattingInstructions}\n${customPrompt}`.trim();
163
+ }
164
+
165
+ private onTickBehavior = () => {
166
+ if (!this.isSpawned || !this.world) return;
167
+ this.behaviors.forEach((b) => b.onUpdate(this, this.world!));
168
+ };
169
+
170
+ public addBehavior(behavior: AgentBehavior) {
171
+ this.behaviors.push(behavior);
172
+ }
173
+
174
+ public getBehaviors(): AgentBehavior[] {
175
+ return this.behaviors;
176
+ }
177
+
178
+ public getNearbyEntities(radius: number = 10): NearbyEntity[] {
179
+ if (!this.world) return [];
180
+ return this.world.entityManager
181
+ .getAllEntities()
182
+ .filter((entity) => entity !== this)
183
+ .map((entity) => {
184
+ const distance = Vector3.fromVector3Like(
185
+ this.position
186
+ ).distance(Vector3.fromVector3Like(entity.position));
187
+ if (distance <= radius) {
188
+ return {
189
+ name:
190
+ entity instanceof PlayerEntity
191
+ ? entity.player.username
192
+ : entity.name,
193
+ type:
194
+ entity instanceof PlayerEntity
195
+ ? "Player"
196
+ : entity instanceof BaseAgent
197
+ ? "Agent"
198
+ : "Entity",
199
+ distance,
200
+ position: entity.position,
201
+ };
202
+ }
203
+ return null;
204
+ })
205
+ .filter((e): e is NearbyEntity => e !== null)
206
+ .sort((a, b) => a.distance - b.distance);
207
+ }
208
+
209
+ public getCurrentState(): Record<string, any> {
210
+ const state: Record<string, any> = {};
211
+ this.behaviors.forEach((behavior) => {
212
+ if (behavior.getState) {
213
+ state[behavior.constructor.name] = behavior.getState();
214
+ }
215
+ });
216
+
217
+ // Add inventory to state
218
+ state.inventory = Array.from(this.inventory.values());
219
+ return state;
220
+ }
221
+
222
+ /**
223
+ * Main chat method. We feed environment/player messages + state to the LLM.
224
+ * We instruct it to produce <monologue> for hidden thoughts, <action type="..."> for real actions.
225
+ */
226
+ public async chat(options: ChatOptions) {
227
+ // Reset inactivity timer when anyone talks nearby
228
+ if (options.type === "Player" || options.type === "Agent") {
229
+ this.lastActionTime = Date.now();
230
+ }
231
+
232
+ // If this is an agent message, delay and allow for interruption
233
+ if (options.type === "Agent") {
234
+ // Clear any pending response
235
+ if (this.pendingAgentResponse) {
236
+ clearTimeout(this.pendingAgentResponse.timeoutId);
237
+ }
238
+
239
+ // Set up new delayed response
240
+ this.pendingAgentResponse = {
241
+ timeoutId: setTimeout(() => {
242
+ this.processChatMessage(options);
243
+ this.pendingAgentResponse = undefined;
244
+ }, 5000), // 5 second delay
245
+ options,
246
+ };
247
+ return;
248
+ }
249
+
250
+ // For player or environment messages, process immediately
251
+ // and cancel any pending agent responses
252
+ if (this.pendingAgentResponse) {
253
+ clearTimeout(this.pendingAgentResponse.timeoutId);
254
+ this.pendingAgentResponse = undefined;
255
+ }
256
+
257
+ await this.processChatMessage(options);
258
+ }
259
+
260
+ private async processChatMessage(options: ChatOptions) {
261
+ const { type, message, player, agent } = options;
262
+ try {
263
+ if (this.chatHistory.length === 0) {
264
+ this.chatHistory.push({
265
+ role: "system",
266
+ content: this.buildSystemPrompt(this.systemPrompt),
267
+ });
268
+ }
269
+
270
+ const currentState = this.getCurrentState();
271
+ const nearbyEntities = this.getNearbyEntities().map((e) => ({
272
+ name: e.name,
273
+ type: e.type,
274
+ state: e instanceof BaseAgent ? e.getCurrentState() : undefined,
275
+ }));
276
+
277
+ let prefix = "";
278
+ if (type === "Environment") prefix = "ENVIRONMENT: ";
279
+ else if (type === "Player" && player)
280
+ prefix = `[${player.username}]: `;
281
+ else if (type === "Agent" && agent)
282
+ prefix = `[${agent.name} (AI)]: `;
283
+
284
+ const userMessage = `${prefix}${message}\nState: ${JSON.stringify(
285
+ currentState
286
+ )}\nNearby: ${JSON.stringify(nearbyEntities)}`;
287
+
288
+ this.chatHistory.push({ role: "user", content: userMessage });
289
+
290
+ const completion = await this.openai.chat.completions.create({
291
+ model: "gpt-4o",
292
+ messages: this.chatHistory,
293
+ temperature: 0.7,
294
+ });
295
+
296
+ const response = completion.choices[0]?.message;
297
+ if (!response || !response.content) return;
298
+
299
+ console.log("Response:", response.content);
300
+
301
+ this.parseXmlResponse(response.content);
302
+
303
+ // Keep the assistant's text in chat history
304
+ this.chatHistory.push({
305
+ role: "assistant",
306
+ content: response.content || "",
307
+ });
308
+ } catch (error) {
309
+ console.error("OpenAI API error:", error);
310
+ }
311
+ }
312
+
313
+ /**
314
+ * Parse the LLM's response for <monologue> and <action> tags.
315
+ * <monologue> is internal. <action> calls onToolCall from behaviors.
316
+ */
317
+ private parseXmlResponse(text: string) {
318
+ // <monologue> hidden
319
+ const monologueRegex = /<monologue>([\s\S]*?)<\/monologue>/g;
320
+ let monologueMatch;
321
+ while ((monologueMatch = monologueRegex.exec(text)) !== null) {
322
+ const thought = monologueMatch[1]?.trim();
323
+ if (thought) {
324
+ this.internalMonologue.push(thought);
325
+ // Broadcast thought to all players
326
+ if (this.world) {
327
+ const allPlayers = this.world.entityManager
328
+ .getAllEntities()
329
+ .filter((e) => e instanceof PlayerEntity)
330
+ .map((e) => (e as PlayerEntity).player);
331
+
332
+ allPlayers.forEach((player) => {
333
+ player.ui.sendData({
334
+ type: "agentThoughts",
335
+ agents: this.world!.entityManager.getAllEntities()
336
+ .filter((e) => e instanceof BaseAgent)
337
+ .map((e) => ({
338
+ name: e.name,
339
+ lastThought:
340
+ (e as BaseAgent).getLastMonologue() ||
341
+ "Idle",
342
+ inventory: Array.from(
343
+ (e as BaseAgent).getInventory().values()
344
+ ),
345
+ })),
346
+ });
347
+ });
348
+ }
349
+ }
350
+ }
351
+
352
+ // <action type="..."> ... </action>
353
+ const actionRegex = /<action\s+type="([^"]+)">([\s\S]*?)<\/action>/g;
354
+ let actionMatch;
355
+ while ((actionMatch = actionRegex.exec(text)) !== null) {
356
+ const actionType = actionMatch[1];
357
+ const actionBody = actionMatch[2]?.trim();
358
+ try {
359
+ console.log("Action:", actionType, actionBody);
360
+ if (!actionBody || actionBody === "{}") {
361
+ this.handleToolCall(actionType, {});
362
+ } else {
363
+ const parsed = JSON.parse(actionBody);
364
+ this.handleToolCall(actionType, parsed);
365
+ }
366
+ this.lastActionTime = Date.now(); // Update last action time
367
+ } catch (e) {
368
+ console.error(`Failed to parse action ${actionType}:`, e);
369
+ console.error("Body:", actionBody);
370
+ }
371
+ }
372
+ }
373
+
374
+ /**
375
+ * Same handleToolCall as in your code. We simply pass the calls to behaviors.
376
+ */
377
+ public handleToolCall(toolName: string, args: any, player?: Player) {
378
+ if (!this.world) return;
379
+ let results: string[] = [];
380
+ console.log("Handling tool call:", toolName, args);
381
+ this.behaviors.forEach((b) => {
382
+ if (b.onToolCall) {
383
+ const result = b.onToolCall(
384
+ this,
385
+ this.world!,
386
+ toolName,
387
+ args,
388
+ player
389
+ );
390
+ if (result) results.push(`${toolName}: ${result}`);
391
+ }
392
+ });
393
+ return results.join("\n");
394
+ }
395
+
396
+ public spawn(world: World, position: Vector3) {
397
+ super.spawn(world, position);
398
+ this.chatUI.load(world);
399
+ }
400
+
401
+ public handleEnvironmentTrigger(message: string) {
402
+ console.log(
403
+ "Environment trigger for agent " + this.name + ":",
404
+ message
405
+ );
406
+ this.chat({
407
+ type: "Environment",
408
+ message,
409
+ });
410
+ }
411
+
412
+ // Clean up interval when agent is destroyed
413
+ public despawn(): void {
414
+ if (this.inactivityCheckInterval) {
415
+ clearInterval(this.inactivityCheckInterval);
416
+ }
417
+ super.despawn();
418
+ }
419
+
420
+ // Add method to get last monologue
421
+ public getLastMonologue(): string | undefined {
422
+ return this.internalMonologue[this.internalMonologue.length - 1];
423
+ }
424
+
425
+ public setChatUIState(state: Record<string, any>) {
426
+ this.chatUI.setState(state);
427
+ }
428
+
429
+ public addToInventory(item: InventoryItem): void {
430
+ const existing = this.inventory.get(item.name);
431
+ if (existing) {
432
+ existing.quantity += item.quantity;
433
+ if (item.metadata) {
434
+ existing.metadata = { ...existing.metadata, ...item.metadata };
435
+ }
436
+ } else {
437
+ this.inventory.set(item.name, { ...item });
438
+ }
439
+ this.broadcastInventoryUpdate();
440
+ }
441
+
442
+ public removeFromInventory(itemName: string, quantity: number): boolean {
443
+ const item = this.inventory.get(itemName);
444
+ if (!item || item.quantity < quantity) return false;
445
+
446
+ item.quantity -= quantity;
447
+ if (item.quantity <= 0) {
448
+ this.inventory.delete(itemName);
449
+ }
450
+ this.broadcastInventoryUpdate();
451
+ return true;
452
+ }
453
+
454
+ public getInventory(): Map<string, InventoryItem> {
455
+ return this.inventory;
456
+ }
457
+
458
+ private broadcastInventoryUpdate(): void {
459
+ if (!this.world) return;
460
+
461
+ const allPlayers = this.world.entityManager
462
+ .getAllEntities()
463
+ .filter((e) => e instanceof PlayerEntity)
464
+ .map((e) => (e as PlayerEntity).player);
465
+
466
+ allPlayers.forEach((player) => {
467
+ player.ui.sendData({
468
+ type: "agentThoughts",
469
+ agents: this.world!.entityManager.getAllEntities()
470
+ .filter((e) => e instanceof BaseAgent)
471
+ .map((e) => ({
472
+ name: e.name,
473
+ lastThought:
474
+ (e as BaseAgent).getLastMonologue() || "Idle",
475
+ inventory: Array.from(
476
+ (e as BaseAgent).getInventory().values()
477
+ ),
478
+ })),
479
+ });
480
+ });
481
+ }
482
+ }
@@ -0,0 +1,181 @@
1
+ import { Vector3, World } from "hytopia";
2
+ import { BaseAgent, type AgentBehavior } from "../BaseAgent";
3
+
4
+ interface FishResult {
5
+ success: boolean;
6
+ size?: "small" | "medium" | "large";
7
+ }
8
+
9
+ /**
10
+ * This is a simple implementation of a fishing behavior for Agents.
11
+ * It does not include animations or any other fancy features.
12
+ * Agents can call actions like `cast_rod` to start fishing, and the environment will trigger a callback when the fishing is complete.
13
+ * This is a simple example of ENVIRONMENT type messages for Agents.
14
+ */
15
+ export class FishingBehavior implements AgentBehavior {
16
+ private isFishing: boolean = false;
17
+ private readonly PIER_LOCATION = new Vector3(31.5, 3, 59.5);
18
+ private readonly FISHING_RANGE = 5; // meters
19
+ private readonly CATCH_PROBABILITIES = {
20
+ nothing: 0.4,
21
+ small: 0.3,
22
+ medium: 0.2,
23
+ large: 0.1,
24
+ };
25
+
26
+ onUpdate(agent: BaseAgent, world: World): void {
27
+ // Could add ambient fishing animations here if needed
28
+ }
29
+
30
+ private isNearPier(agent: BaseAgent): boolean {
31
+ const distance = Vector3.fromVector3Like(agent.position).distance(
32
+ this.PIER_LOCATION
33
+ );
34
+ return distance <= this.FISHING_RANGE;
35
+ }
36
+
37
+ private rollForFish(): FishResult {
38
+ const roll = Math.random();
39
+ let cumulative = 0;
40
+
41
+ // Check for no catch
42
+ cumulative += this.CATCH_PROBABILITIES.nothing;
43
+ if (roll < cumulative) {
44
+ return { success: false };
45
+ }
46
+
47
+ // Check for small fish
48
+ cumulative += this.CATCH_PROBABILITIES.small;
49
+ if (roll < cumulative) {
50
+ return {
51
+ success: true,
52
+ size: "small",
53
+ };
54
+ }
55
+
56
+ // Check for medium fish
57
+ cumulative += this.CATCH_PROBABILITIES.medium;
58
+ if (roll < cumulative) {
59
+ return {
60
+ success: true,
61
+ size: "medium",
62
+ };
63
+ }
64
+
65
+ // Must be a large fish
66
+ return {
67
+ success: true,
68
+ size: "large",
69
+ };
70
+ }
71
+
72
+ onToolCall(
73
+ agent: BaseAgent,
74
+ world: World,
75
+ toolName: string,
76
+ args: any
77
+ ): string | void {
78
+ if (toolName === "cast_rod") {
79
+ console.log("Fishing tool called");
80
+
81
+ if (!this.isNearPier(agent)) {
82
+ return "You need to be closer to the pier to fish!";
83
+ }
84
+
85
+ if (this.isFishing) {
86
+ return "You're already fishing!";
87
+ }
88
+
89
+ this.isFishing = true;
90
+
91
+ // Start fishing animation if available
92
+ agent.stopModelAnimations(["walk", "run"]);
93
+ agent.startModelLoopedAnimations(["idle"]); // Could be replaced with a fishing animation
94
+
95
+ // Simulate fishing time
96
+ setTimeout(() => {
97
+ this.isFishing = false;
98
+ const result = this.rollForFish();
99
+
100
+ if (!result.success) {
101
+ agent.handleEnvironmentTrigger(
102
+ "Nothing seems to be biting..."
103
+ );
104
+ return;
105
+ }
106
+
107
+ const fishDescription = `${result.size} fish`;
108
+ agent.addToInventory({
109
+ name: fishDescription,
110
+ quantity: 1,
111
+ metadata: {
112
+ size: result.size,
113
+ },
114
+ });
115
+
116
+ agent.handleEnvironmentTrigger(
117
+ `You caught ${fishDescription}!`
118
+ );
119
+ }, 5000); // 5 second fishing time
120
+
121
+ return "Casting your line...";
122
+ } else if (toolName === "give_fish") {
123
+ const { size, weight, target } = args;
124
+ const fishDescription = `${size} fish`;
125
+
126
+ if (!agent.removeFromInventory(fishDescription, 1)) {
127
+ return "You don't have that fish anymore!";
128
+ }
129
+
130
+ const nearbyEntities = agent.getNearbyEntities(5);
131
+ const targetEntity = nearbyEntities.find((e) => e.name === target);
132
+
133
+ if (!targetEntity) {
134
+ return `Cannot find ${target} nearby. Try getting closer to them.`;
135
+ }
136
+
137
+ // Add to target's inventory if it's an agent
138
+ if (targetEntity.type === "Agent") {
139
+ const targetAgent = world.entityManager
140
+ .getAllEntities()
141
+ .find(
142
+ (e) => e instanceof BaseAgent && e.name === target
143
+ ) as BaseAgent;
144
+
145
+ if (targetAgent) {
146
+ targetAgent.addToInventory({
147
+ name: fishDescription,
148
+ quantity: 1,
149
+ metadata: { size, weight },
150
+ });
151
+ }
152
+ }
153
+
154
+ return `Successfully gave ${fishDescription} to ${target}`;
155
+ }
156
+ }
157
+
158
+ getPromptInstructions(): string {
159
+ return `
160
+ To fish at the pier, use:
161
+ <action type="cast_rod"></action>
162
+
163
+ You must call cast_rod exactly like this, with the empty object inside the action tag.
164
+
165
+ To give a fish to another agent, use:
166
+ <action type="give_fish">
167
+ {
168
+ size: "small" | "medium" | "large",
169
+ target: "name of the player or agent to give the fish to"
170
+ }
171
+ </action>
172
+
173
+ You must be within 5 meters of the pier to fish.
174
+ Each attempt takes 5 seconds and has a chance to catch nothing or a fish of varying sizes.
175
+ You can only have one line in the water at a time.`;
176
+ }
177
+
178
+ getState(): string {
179
+ return this.isFishing ? "Currently fishing" : "Not fishing";
180
+ }
181
+ }