mindcraft 0.1.4-0

package/src/agent/vision/camera.js

@@ -0,0 +1,78 @@
+import { Viewer } from 'prismarine-viewer/viewer/lib/viewer.js';
+import { WorldView } from 'prismarine-viewer/viewer/lib/worldView.js';
+import { getBufferFromStream } from 'prismarine-viewer/viewer/lib/simpleUtils.js';
+
+import THREE from 'three';
+import { createCanvas } from 'node-canvas-webgl/lib/index.js';
+import fs from 'fs/promises';
+import { Vec3 } from 'vec3';
+import { EventEmitter } from 'events';
+
+import worker_threads from 'worker_threads';
+global.Worker = worker_threads.Worker;
+
+
+export class Camera extends EventEmitter {
+    constructor (bot, fp) {
+        super();
+        this.bot = bot;
+        this.fp = fp;
+        this.viewDistance = 12;
+        this.width = 800;
+        this.height = 512;
+        this.canvas = createCanvas(this.width, this.height);
+        this.renderer = new THREE.WebGLRenderer({ canvas: this.canvas });
+        this.viewer = new Viewer(this.renderer);
+        this._init().then(() => {
+            this.emit('ready');
+        })
+    }
+  
+    async _init () {
+        const botPos = this.bot.entity.position;
+        const center = new Vec3(botPos.x, botPos.y+this.bot.entity.height, botPos.z);
+        this.viewer.setVersion(this.bot.version);
+        // Load world
+        const worldView = new WorldView(this.bot.world, this.viewDistance, center);
+        this.viewer.listen(worldView);
+        worldView.listenToBot(this.bot);
+        await worldView.init(center);
+        this.worldView = worldView;
+    }
+  
+    async capture() {
+        const center = new Vec3(this.bot.entity.position.x, this.bot.entity.position.y+this.bot.entity.height, this.bot.entity.position.z);
+        this.viewer.camera.position.set(center.x, center.y, center.z);
+        await this.worldView.updatePosition(center);
+        this.viewer.setFirstPersonCamera(this.bot.entity.position, this.bot.entity.yaw, this.bot.entity.pitch);
+        this.viewer.update();
+        this.renderer.render(this.viewer.scene, this.viewer.camera);
+
+        const imageStream = this.canvas.createJPEGStream({
+            bufsize: 4096,
+            quality: 100,
+            progressive: false
+        });
+        
+        const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+        const filename = `screenshot_${timestamp}`;
+
+        const buf = await getBufferFromStream(imageStream);
+        await this._ensureScreenshotDirectory();
+        await fs.writeFile(`${this.fp}/${filename}.jpg`, buf);
+        console.log('saved', filename);
+        return filename;
+    }
+
+    async _ensureScreenshotDirectory() {
+        let stats;
+        try {
+            stats = await fs.stat(this.fp);
+        } catch (e) {
+            if (!stats?.isDirectory()) {
+                await fs.mkdir(this.fp);
+            }
+        }
+    }
+}
+  

package/src/agent/vision/vision_interpreter.js

@@ -0,0 +1,82 @@
+import { Vec3 } from 'vec3';
+import { Camera } from "./camera.js";
+import fs from 'fs';
+import settings from '../settings.js';
+
+export class VisionInterpreter {
+    constructor(agent, allow_vision) {
+        this.agent = agent;
+        this.allow_vision = allow_vision;
+        this.fp = `${settings.data_dir}/${agent.name}/screenshots/`;
+        if (allow_vision) {
+            this.camera = new Camera(agent.bot, this.fp);
+        }
+    }
+
+    async lookAtPlayer(player_name, direction) {
+        if (!this.allow_vision || !this.agent.prompter.vision_model.sendVisionRequest) {
+            return "Vision is disabled. Use other methods to describe the environment.";
+        }
+        let result = "";
+        const bot = this.agent.bot;
+        const player = bot.players[player_name]?.entity;
+        if (!player) {
+            return `Could not find player ${player_name}`;
+        }
+
+        let filename;
+        if (direction === 'with') {
+            await bot.look(player.yaw, player.pitch);
+            result = `Looking in the same direction as ${player_name}\n`;
+            filename = await this.camera.capture();
+        } else {
+            await bot.lookAt(new Vec3(player.position.x, player.position.y + player.height, player.position.z));
+            result = `Looking at player ${player_name}\n`;
+            filename = await this.camera.capture();
+
+        }
+
+        return result + `Image analysis: "${await this.analyzeImage(filename)}"`;
+    }
+
+    async lookAtPosition(x, y, z) {
+        if (!this.allow_vision || !this.agent.prompter.vision_model.sendVisionRequest) {
+            return "Vision is disabled. Use other methods to describe the environment.";
+        }
+        let result = "";
+        const bot = this.agent.bot;
+        await bot.lookAt(new Vec3(x, y + 2, z));
+        result = `Looking at coordinate ${x}, ${y}, ${z}\n`;
+
+        let filename = await this.camera.capture();
+
+        return result + `Image analysis: "${await this.analyzeImage(filename)}"`;
+    }
+
+    getCenterBlockInfo() {
+        const bot = this.agent.bot;
+        const maxDistance = 128; // Maximum distance to check for blocks
+        const targetBlock = bot.blockAtCursor(maxDistance);
+        
+        if (targetBlock) {
+            return `Block at center view: ${targetBlock.name} at (${targetBlock.position.x}, ${targetBlock.position.y}, ${targetBlock.position.z})`;
+        } else {
+            return "No block in center view";
+        }
+    }
+
+    async analyzeImage(filename) {
+        try {
+            const imageBuffer = fs.readFileSync(`${this.fp}/${filename}.jpg`);
+            const messages = this.agent.history.getHistory();
+
+            const blockInfo = this.getCenterBlockInfo();
+            const result = await this.agent.prompter.promptVision(messages, imageBuffer);
+            return result + `\n${blockInfo}`;
+
+        } catch (error) {
+            console.warn('Error reading image:', error);
+            return `Error reading image: ${error.message}`;
+        }
+    }
+} 

package/src/mindcraft/index.js

@@ -0,0 +1,28 @@
+import { readFileSync } from 'fs';
+import { fileURLToPath } from 'url';
+import defaultSettings from '../../settings.js';
+
+export {
+    init,
+    createAgent,
+    getAgentProcess,
+    startAgent,
+    stopAgent,
+    destroyAgent,
+    shutdown,
+} from './mindcraft.js';
+
+export { createMindServer, registerAgent, getIO, getServer } from './mindserver.js';
+
+export { defaultSettings };
+
+const settings_spec_path = fileURLToPath(new URL('./public/settings_spec.json', import.meta.url));
+export const settingsSpec = JSON.parse(readFileSync(settings_spec_path, 'utf8'));
+
+/**
+ * Build a minimal Claude profile. Omits `embedding` so mindcraft falls back to
+ * word-overlap similarity, meaning only ANTHROPIC_API_KEY is required.
+ */
+export function claudeProfile({ name, model = 'claude-sonnet-4-6' }) {
+    return { name, model };
+}

package/src/mindcraft/mcserver.js

@@ -0,0 +1,154 @@
+import net from 'net';
+import mc from 'minecraft-protocol';
+
+/**
+ * Scans the IP address for Minecraft LAN servers and collects their info.
+ * @param {string} ip - The IP address to scan.
+ * @param {number} port - The port to check.
+ * @param {number} timeout - The connection timeout in ms.
+ * @param {boolean} verbose - Whether to print output on connection errors.
+ * @returns {Promise<Array>} - A Promise that resolves to an array of server info objects.
+ */
+export async function serverInfo(ip, port, timeout = 1000, verbose = false) {
+    return new Promise((resolve) => {
+
+        let timeoutId = setTimeout(() => {
+            if (verbose)
+                console.error(`Timeout pinging server ${ip}:${port}`);
+            resolve(null); // Resolve as null if no response within timeout
+        }, timeout);
+
+        mc.ping({
+            host: ip,
+            port
+        }, (err, response) => {
+            clearTimeout(timeoutId);
+
+            if (err) {
+                if (verbose)
+                    console.error(`Error pinging server ${ip}:${port}`, err);
+                return resolve(null);
+            }
+
+            // extract version number from modded servers like "Paper 1.21.4"
+            const version = response?.version?.name || '';
+            const match = String(version).match(/\d+\.\d+(?:\.\d+)?/);
+            const numericVersion = match ? match[0] : null;
+            if (numericVersion !== version) {
+                console.log(`Modded server found (${version}), attempting to use ${numericVersion}...`);
+            }
+
+            const serverInfo = {
+                host: ip,
+                port,
+                name: response.description.text || 'No description provided.',
+                ping: response.latency,
+                version: numericVersion
+            };
+
+            resolve(serverInfo);
+        });
+    });
+}
+
+/**
+ * Scans the IP address for Minecraft LAN servers and collects their info.
+ * @param {string} ip - The IP address to scan.
+ * @param {boolean} earlyExit - Whether to exit early after finding a server.
+ * @param {number} timeout - The connection timeout in ms.
+ * @returns {Promise<Array>} - A Promise that resolves to an array of server info objects.
+ */
+export async function findServers(ip, earlyExit = false, timeout = 100) {
+    const servers = [];
+    const startPort = 49000;
+    const endPort = 65000;
+
+    const checkPort = (port) => {
+        return new Promise((resolve) => {
+            const socket = net.createConnection({ host: ip, port, timeout }, () => {
+                socket.end();
+                resolve(port); // Port is open
+            });
+
+            socket.on('error', () => resolve(null)); // Port is closed
+            socket.on('timeout', () => {
+                socket.destroy();
+                resolve(null);
+            });
+        });
+    };
+
+    // This supresses a lot of annoying console output from the mc library
+    // TODO: find a better way to do this, it supresses other useful output
+    const originalConsoleLog = console.log;
+    console.log = () => { };
+    
+    for (let port = startPort; port <= endPort; port++) {
+        const openPort = await checkPort(port);
+        if (openPort) {
+            const server = await serverInfo(ip, port, 200, false);
+            if (server) {
+                servers.push(server);
+
+                if (earlyExit) break;
+            }
+        }
+    }
+
+    // Restore console output
+    console.log = originalConsoleLog;
+
+    return servers;
+}
+
+/**
+ * Gets the MC server info from the host and port.
+ * @param {string} host - The host to search for.
+ * @param {number} port - The port to search for.
+ * @param {string} version - The version to search for.
+ * @returns {Promise<Object>} - A Promise that resolves to the server info object.
+ */
+export async function getServer(host, port, version) {
+    let server = null;
+    let serverString = "";
+    let serverVersion = "";
+    
+    // Search for server
+    if (port == -1)
+    {
+        console.log(`No port provided. Searching for LAN server on host ${host}...`);
+        
+        await findServers(host, true).then((servers) => {
+            if (servers.length > 0)
+                server = servers[0];
+        });
+
+        if (server == null)
+            throw new Error(`No server found on LAN.`);
+    }
+    else
+        server = await serverInfo(host, port, 1000, true);
+
+    // Server not found
+    if (server == null) 
+        throw new Error(`MC server not found. (Host: ${host}, Port: ${port}) Check the host and port in settings.js, and ensure the server is running and open to public or LAN.`);
+
+    serverString = `(Host: ${server.host}, Port: ${server.port}, Version: ${server.version})`;
+
+    if (version === "auto") 
+        serverVersion = server.version;
+    else
+        serverVersion = version;
+    // Server version unsupported / mismatch
+    const isSupported = mc.supportedVersions.some(v => 
+        serverVersion === v || (serverVersion.startsWith(v) && serverVersion.charAt(v.length) === '.')
+    ); // Checks version or parent version (e.g. if 1.7 is supported then 1.7.2 will be allowed)
+     if (!isSupported)
+        throw new Error(`MC server was found ${serverString}, but version is unsupported. Supported versions are: ${mc.supportedVersions.join(", ")}.`);
+    else if (version !== "auto" && server.version !== version)
+        throw new Error(`MC server was found ${serverString}, but version is incorrect. Expected ${version}, but found ${server.version}. Check the server version in settings.js.`);
+    else
+        console.log(`MC server found. ${serverString}`);
+
+    return server;
+}

package/src/mindcraft/mindcraft.js

@@ -0,0 +1,111 @@
+import { createMindServer, registerAgent, numStateListeners } from './mindserver.js';
+import { AgentProcess } from '../process/agent_process.js';
+import { getServer } from './mcserver.js';
+import open from 'open';
+
+let mindserver;
+let connected = false;
+let agent_processes = {};
+let agent_count = 0;
+let mindserver_port = 8080;
+
+export async function init(host_public=false, port=8080, auto_open_ui=true) {
+    if (connected) {
+        console.error('Already initiliazed!');
+        return;
+    }
+    mindserver = createMindServer(host_public, port);
+    mindserver_port = port;
+    connected = true;
+    if (auto_open_ui) {
+        setTimeout(() => {
+            // check if browser listener is already open
+            if (numStateListeners() === 0) {
+                open('http://localhost:'+port);
+            }
+        }, 3000);
+    }
+}
+
+export async function createAgent(settings) {
+    if (!settings.profile.name) {
+        console.error('Agent name is required in profile');
+        return {
+            success: false,
+            error: 'Agent name is required in profile'
+        };
+    }
+    settings = JSON.parse(JSON.stringify(settings));
+    let agent_name = settings.profile.name;
+    const agentIndex = agent_count++;
+    const viewer_port = 3000 + agentIndex;
+    registerAgent(settings, viewer_port);
+    let load_memory = settings.load_memory || false;
+    let init_message = settings.init_message || null;
+
+    try {
+        try {
+            const server = await getServer(settings.host, settings.port, settings.minecraft_version);
+            settings.host = server.host;
+            settings.port = server.port;
+            settings.minecraft_version = server.version;
+        } catch (error) {
+            console.warn(`Error getting server:`, error);
+            if (settings.minecraft_version === "auto") {
+                settings.minecraft_version = null;
+            }
+            console.warn(`Attempting to connect anyway...`);
+        }
+
+        const agentProcess = new AgentProcess(agent_name, mindserver_port);
+        agentProcess.start(load_memory, init_message, agentIndex);
+        agent_processes[settings.profile.name] = agentProcess;
+    } catch (error) {
+        console.error(`Error creating agent ${agent_name}:`, error);
+        destroyAgent(agent_name);
+        return {
+            success: false,
+            error: error.message
+        };
+    }
+    return {
+        success: true,
+        error: null
+    };
+}
+
+export function getAgentProcess(agentName) {
+    return agent_processes[agentName];
+}
+
+export function startAgent(agentName) {
+    if (agent_processes[agentName]) {
+        agent_processes[agentName].forceRestart();
+    }
+    else {
+        console.error(`Cannot start agent ${agentName}; not found`);
+    }
+}
+
+export function stopAgent(agentName) {
+    if (agent_processes[agentName]) {
+        agent_processes[agentName].stop();
+    }
+}
+
+export function destroyAgent(agentName) {
+    if (agent_processes[agentName]) {
+        agent_processes[agentName].stop();
+        delete agent_processes[agentName];
+    }
+}
+
+export function shutdown() {
+    console.log('Shutting down');
+    for (let agentName in agent_processes) {
+        agent_processes[agentName].stop();
+    }
+    setTimeout(() => {
+        process.exit(0);
+    }, 2000);
+}