@pheem49/mint 1.5.0 → 1.5.1

package/src/System/tool_registry.js

@@ -0,0 +1,280 @@
+const TOOL_REGISTRY = Object.freeze({
+    none: {
+        permission: 'safe',
+        required: [],
+        description: 'No action.'
+    },
+    web_search: {
+        permission: 'safe',
+        required: ['query'],
+        codeAgentOnly: true,
+        description: 'Search the internet when outside knowledge is required.'
+    },
+    list_files: {
+        permission: 'safe',
+        required: [],
+        codeAgentOnly: true,
+        description: 'List files under a workspace-relative path.'
+    },
+    read_file: {
+        permission: 'safe',
+        required: ['path'],
+        codeAgentOnly: true,
+        description: 'Read a workspace file, optionally bounded by startLine/endLine.'
+    },
+    search_code: {
+        permission: 'safe',
+        required: ['query'],
+        codeAgentOnly: true,
+        description: 'Search text in the workspace.'
+    },
+    find_path: {
+        permission: 'safe',
+        required: ['query'],
+        chatAction: true,
+        description: 'Find files or folders by name.'
+    },
+    run_shell: {
+        permission: 'approval',
+        required: ['command'],
+        codeAgentOnly: true,
+        important: true,
+        description: 'Run a non-destructive shell command after user approval.'
+    },
+    apply_patch: {
+        permission: 'approval',
+        required: ['patch'],
+        codeAgentOnly: true,
+        important: true,
+        description: 'Patch an existing file after user approval.'
+    },
+    write_file: {
+        permission: 'approval',
+        required: ['path', 'content'],
+        codeAgentOnly: true,
+        important: true,
+        description: 'Create or replace a file after user approval.'
+    },
+    ask_user: {
+        permission: 'safe',
+        required: ['question'],
+        codeAgentOnly: true,
+        description: 'Ask the user for clarification.'
+    },
+    open_url: {
+        permission: 'safe',
+        required: ['target'],
+        chatAction: true,
+        description: 'Open a URL.'
+    },
+    search: {
+        permission: 'safe',
+        required: ['target'],
+        chatAction: true,
+        description: 'Open a web search.'
+    },
+    open_app: {
+        permission: 'safe',
+        required: ['target'],
+        chatAction: true,
+        description: 'Open a local application.'
+    },
+    web_automation: {
+        permission: 'safe',
+        required: ['target'],
+        chatAction: true,
+        important: true,
+        description: 'Perform browser automation.'
+    },
+    create_folder: {
+        permission: 'safe',
+        required: ['target'],
+        chatAction: true,
+        description: 'Create a folder.'
+    },
+    open_file: {
+        permission: 'safe',
+        required: ['target'],
+        chatAction: true,
+        description: 'Open a local file.'
+    },
+    open_folder: {
+        permission: 'safe',
+        required: ['target'],
+        chatAction: true,
+        description: 'Open a local folder.'
+    },
+    delete_file: {
+        permission: 'dangerous',
+        required: ['target'],
+        chatAction: true,
+        important: true,
+        description: 'Delete a file only after explicit dangerous-action permission.'
+    },
+    clipboard_write: {
+        permission: 'safe',
+        required: ['target'],
+        chatAction: true,
+        description: 'Write text to clipboard.'
+    },
+    learn_file: {
+        permission: 'safe',
+        required: ['target'],
+        chatAction: true,
+        description: 'Index a file into the knowledge base.'
+    },
+    learn_folder: {
+        permission: 'safe',
+        required: ['target'],
+        chatAction: true,
+        description: 'Index a folder into the knowledge base.'
+    },
+    system_info: {
+        permission: 'safe',
+        required: [],
+        chatAction: true,
+        description: 'Read local system info, or weather when target is a city.'
+    },
+    plugin: {
+        permission: 'safe',
+        required: ['pluginName', 'target'],
+        chatAction: true,
+        description: 'Run a Mint plugin.'
+    },
+    mcp_tool: {
+        permission: 'safe',
+        required: ['server', 'target'],
+        chatAction: true,
+        description: 'Call an MCP tool.'
+    },
+    mouse_click: {
+        permission: 'safe',
+        required: ['x', 'y'],
+        chatAction: true,
+        important: true,
+        description: 'Click at screen coordinates.'
+    },
+    mouse_move: {
+        permission: 'safe',
+        required: ['x', 'y'],
+        chatAction: true,
+        description: 'Move the mouse.'
+    },
+    type_text: {
+        permission: 'safe',
+        required: ['target'],
+        chatAction: true,
+        important: true,
+        description: 'Type text into the active UI.'
+    },
+    key_tap: {
+        permission: 'safe',
+        required: ['target'],
+        chatAction: true,
+        important: true,
+        description: 'Press a key.'
+    },
+    system_automation: {
+        permission: 'approval',
+        required: ['target'],
+        chatAction: true,
+        important: true,
+        description: 'Change system settings after approval.'
+    },
+    finish: {
+        permission: 'safe',
+        required: ['summary'],
+        codeAgentOnly: true,
+        description: 'Finish the task and reply.'
+    }
+});
+
+function getTool(name) {
+    return TOOL_REGISTRY[name] || null;
+}
+
+function listToolNames(filter = {}) {
+    return Object.entries(TOOL_REGISTRY)
+        .filter(([, tool]) => {
+            if (filter.chatAction === true && tool.chatAction !== true) return false;
+            if (filter.codeAgent === true && tool.chatAction === true && tool.codeAgentOnly !== true) return true;
+            return true;
+        })
+        .map(([name]) => name);
+}
+
+function listChatActionNames() {
+    return Object.entries(TOOL_REGISTRY)
+        .filter(([, tool]) => tool.chatAction === true)
+        .map(([name]) => name);
+}
+
+function listCodeAgentActionNames() {
+    return Object.entries(TOOL_REGISTRY)
+        .filter(([, tool]) => tool.codeAgentOnly === true || tool.chatAction === true || tool.required)
+        .map(([name]) => name);
+}
+
+function isEmptyToolValue(value) {
+    if (value === undefined || value === null) return true;
+    if (typeof value === 'string') return value.trim() === '';
+    if (Array.isArray(value)) return value.length === 0;
+    if (typeof value === 'object') return Object.keys(value).length === 0;
+    return false;
+}
+
+function validateToolInput(action, input = {}) {
+    const tool = getTool(action);
+    if (!tool) {
+        throw new Error(`Unsupported action: ${action}`);
+    }
+
+    const missing = (tool.required || []).filter(field => {
+        if (!isEmptyToolValue(input[field])) return false;
+        if (field === 'target' && (!isEmptyToolValue(input.path) || !isEmptyToolValue(input.query))) return false;
+        if (field === 'query' && !isEmptyToolValue(input.target)) return false;
+        return true;
+    });
+    if (missing.length > 0) {
+        throw new Error(`Action "${action}" is missing required input field(s): ${missing.join(', ')}`);
+    }
+
+    if (action === 'apply_patch') {
+        const patchInput = input.patch || {};
+        if (!patchInput.path || !Array.isArray(patchInput.hunks) || patchInput.hunks.length === 0) {
+            throw new Error('Action "apply_patch" requires input.patch.path and at least one hunk.');
+        }
+    }
+
+    return tool;
+}
+
+function isImportantAction(action) {
+    const tool = getTool(action);
+    return !!(tool && tool.important);
+}
+
+function buildChatActionTypeUnion() {
+    return ['none', ...listChatActionNames()].filter((name, index, arr) => arr.indexOf(name) === index).map(name => `"${name}"`).join(' | ');
+}
+
+function buildToolPromptSection() {
+    const lines = ['\n\nAVAILABLE BUILT-IN ACTIONS:'];
+    for (const name of listChatActionNames()) {
+        const tool = getTool(name);
+        lines.push(`- ${name}: ${tool.description}`);
+    }
+    return lines.join('\n');
+}
+
+module.exports = {
+    TOOL_REGISTRY,
+    getTool,
+    listToolNames,
+    listChatActionNames,
+    listCodeAgentActionNames,
+    validateToolInput,
+    isImportantAction,
+    buildChatActionTypeUnion,
+    buildToolPromptSection
+};

package/src/System/window_manager.js

@@ -10,8 +10,10 @@ function createWindowManager(projectRoot) {
 
     function createMainWindow() {
         mainWindow = new BrowserWindow({
-            width: 600,
-            height: 800,
+            width: 1180,
+            height: 860,
+            minWidth: 900,
+            minHeight: 680,
             icon: path.join(projectRoot, 'assets', 'icon.png'),
             webPreferences: {
                 preload: path.join(projectRoot, 'preload.js'),

package/src/UI/live2d_manager.js

@@ -0,0 +1,368 @@
+/**
+ * Live2DManager - Encapsulates Live2D model loading, fitting, and lip-sync logic.
+ */
+window.Live2DManager = {
+    app: null,
+    model: null,
+    resizeObserver: null,
+    lipSyncInterval: null,
+    expIndex: 0,
+    lastInteractionAt: 0,
+    expressionToastTimeout: null,
+    expressionParamIds: [
+        'Param54',
+        'Param55',
+        'Param68',
+        'Param76',
+        'Param91',
+        'Param93',
+        'Param94',
+        'Param96',
+        'ParamAngleY',
+        'ParamAngleZ',
+        'ParamEyeBallX',
+        'ParamEyeBallY',
+        'ParamMouthForm',
+        'ParamMouthOpenY'
+    ],
+    expressionNames: [
+        { id: null, label: 'Normal' },
+        { id: 'Apron', label: 'Apron' },
+        { id: 'Dazed', label: 'Dazed' },
+        { id: 'Photo', label: 'Photo' },
+        { id: 'Glasses', label: 'Glasses' },
+        { id: 'Pen', label: 'Writing' },
+        { id: 'Click', label: 'Blush' },
+        { id: 'CatFilter', label: 'Cat Ears' },
+        { id: 'DazedEyes', label: 'Dazed Eyes' }
+    ],
+
+    async loadModel(mountEl, statusEl, shellEl) {
+        this.statusEl = statusEl; // Store for later use
+        if (!mountEl) return;
+        if (statusEl) {
+            statusEl.classList.remove('is-error');
+            statusEl.innerHTML = `
+                <div class="loader-dots">
+                    <span></span><span></span><span></span>
+                </div>
+                <div style="font-size: 0.7rem; opacity: 0.8; letter-spacing: 0.05em;">SYNCHRONIZING MINT...</div>
+            `;
+        }
+        if (!window.PIXI || !window.PIXI.live2d) {
+            const message = 'Live2D runtime is not available.';
+            console.error(message);
+            if (statusEl) {
+                statusEl.classList.add('is-error');
+                statusEl.textContent = message;
+            }
+            return;
+        }
+
+        try {
+            window.PIXI.live2d.Live2DModel.registerTicker(window.PIXI.Ticker);
+
+            this.app = new window.PIXI.Application({
+                autoDensity: true,
+                antialias: true,
+                backgroundAlpha: 0,
+                resizeTo: mountEl,
+                resolution: window.devicePixelRatio || 1
+            });
+
+            mountEl.prepend(this.app.view);
+
+            const modelUrl = new URL('../../models/Shiroko_Model/Shiroko/Shiroko_Core/%E9%9D%A2%E9%A5%BC0.model3.json', window.location.href).href;
+            this.model = await window.PIXI.live2d.Live2DModel.from(modelUrl, {
+                autoInteract: true
+            });
+            this.expressionToastEl = document.getElementById('expression-toast');
+
+            this.model.anchor.set(0.5, 0.5);
+            this.app.stage.addChild(this.model);
+
+            // -- Interaction Setup --
+            this.model.interactive = true;
+            this.model.buttonMode = true;
+
+            // Tap Interaction. This model does not define Cubism HitAreas, so use
+            // normalized model coordinates to provide stable region reactions.
+            this.model.on('pointertap', (e) => this.handleModelTap(e));
+            this.model.on('hit', (hitAreaNames) => {
+                console.log(`[Live2D] Runtime hit detected: ${hitAreaNames}`);
+            });
+
+            const fitModel = () => {
+                if (!this.model || !mountEl) return;
+                const mountWidth = mountEl.clientWidth || 460;
+                const mountHeight = mountEl.clientHeight || 620;
+                this.app.renderer.resize(mountWidth, mountHeight);
+
+                const internal = this.model.internalModel || {};
+                const modelWidth = internal.width || internal.originalWidth || this.model.width || 1;
+                const modelHeight = internal.height || internal.originalHeight || this.model.height || 1;
+                const widthScale = mountWidth / Math.max(modelWidth, 1);
+                const heightScale = mountHeight / Math.max(modelHeight, 1);
+                
+                // Reduced zoom to 2.0 as requested
+                const scale = Math.min(widthScale, heightScale) * 1.8;
+
+                this.model.scale.set(scale);
+                // Adjusted Y offset to 1.0 as requested
+                this.model.position.set(mountWidth / 2, mountHeight / 2 + mountHeight * 0.6);
+            };
+
+            requestAnimationFrame(() => {
+                fitModel();
+                requestAnimationFrame(fitModel);
+            });
+            this.resizeObserver = new ResizeObserver(fitModel);
+            this.resizeObserver.observe(mountEl);
+
+            shellEl?.classList.add('is-live2d-ready');
+            if (statusEl) statusEl.textContent = '';
+            this.model.motion('Idle', 0).catch(() => {});
+        } catch (error) {
+            console.error('Failed to load Live2D model:', error);
+            shellEl?.classList.remove('is-live2d-ready');
+            if (statusEl) {
+                statusEl.classList.add('is-error');
+                statusEl.textContent = `Live2D failed: ${error && error.message ? error.message : String(error)}`;
+            }
+        }
+    },
+
+    showStatus(text, duration = 2000) {
+        if (!this.statusEl) return;
+        this.statusEl.textContent = text;
+        this.statusEl.style.opacity = '1';
+        
+        if (this.statusTimeout) clearTimeout(this.statusTimeout);
+        this.statusTimeout = setTimeout(() => {
+            this.statusEl.style.opacity = '0';
+            setTimeout(() => {
+                if (this.statusEl.style.opacity === '0') this.statusEl.textContent = '';
+            }, 500);
+        }, duration);
+    },
+
+    handleModelTap(event) {
+        if (!this.model) return;
+
+        const now = Date.now();
+        if (now - this.lastInteractionAt < 3000) return;
+
+        const region = this.getInteractionRegion(event);
+        if (!region) return;
+        this.lastInteractionAt = now;
+        const expressionId = region.expression || 'Click';
+
+        console.log(`[Live2D] Interaction: ${region.id}`, region);
+        this.applyExpression(expressionId);
+        this.showStatus(region.label, 2500);
+
+        window.dispatchEvent(new CustomEvent('live2d-model-interaction', {
+            detail: {
+                region: region.id,
+                label: region.label,
+                prompt: region.prompt
+            }
+        }));
+
+        setTimeout(() => {
+            const currentIdx = this.expIndex === 0 ? 0 : this.expIndex;
+            const prevExp = this.expressionNames[currentIdx]?.id;
+            this.applyExpression(prevExp);
+        }, 2000);
+    },
+
+    getInteractionRegion(event) {
+        try {
+            const point = this.getPointerViewportPoint(event);
+            if (!point) return null;
+            const { x, y } = point;
+
+            if (this.isPointInZone(x, y, 0.37, 0.395, 0.25, 0.13)) {
+                return {
+                    id: 'face',
+                    label: 'Cat Ears',
+                    expression: 'CatFilter',
+                    prompt: 'The user poked Mint model on the cheek. Reply briefly, shyly or with a light tease. Use the same language as the user’s recent conversation; do not switch to Thai unless the user has been speaking Thai.'
+                };
+            }
+
+            if (this.isPointInZone(x, y, 0.35, 0.30, 0.29, 0.09)) {
+                return {
+                    id: 'head',
+                    label: 'Head Pat',
+                    expression: 'Dazed',
+                    prompt: 'The user patted Mint model on the head. Reply briefly in a cute, slightly shy way. Use the same language as the user’s recent conversation; do not switch to Thai unless the user has been speaking Thai.'
+                };
+            }
+
+            const isLeftHand = this.isPointInZone(x, y, 0.17, 0.65, 0.19, 0.17);
+            const isRightHand = this.isPointInZone(x, y, 0.70, 0.67, 0.17, 0.17);
+            if (isLeftHand || isRightHand) {
+                return {
+                    id: isLeftHand ? 'left-hand' : 'right-hand',
+                    label: 'Hand Tap',
+                    expression: 'Pen',
+                    prompt: 'The user tapped Mint model’s hand. Reply briefly as if ready to help or take a request. Use the same language as the user’s recent conversation; do not switch to Thai unless the user has been speaking Thai.'
+                };
+            }
+
+            if (this.isPointInZone(x, y, 0.31, 0.76, 0.38, 0.24)) {
+                return {
+                    id: 'lower-body',
+                    label: 'Careful',
+                    expression: 'Photo',
+                    prompt: 'The user touched the lower body area of Mint model. Reply briefly in a shy, playful way, similar to “hehe~ what are you playing at, that makes me blush,” then gently invite the user back to chatting or work. Use the same language as the user’s recent conversation; do not switch to Thai unless the user has been speaking Thai.'
+                };
+            }
+
+            if (this.isPointInZone(x, y, 0.36, 0.55, 0.29, 0.15)) {
+                return {
+                    id: 'body',
+                    label: 'Shoulder Tap',
+                    expression: 'Click',
+                    prompt: 'The user tapped Mint model’s body or shoulder. Reply briefly as if turning toward the user and asking what they need help with. Use the same language as the user’s recent conversation; do not switch to Thai unless the user has been speaking Thai.'
+                };
+            }
+
+            return null;
+        } catch (error) {
+            console.error('[Live2D] Failed to resolve interaction region:', error);
+            return null;
+        }
+    },
+
+    getPointerViewportPoint(event) {
+        const originalEvent = event?.data?.originalEvent;
+        const rect = this.app?.view?.getBoundingClientRect?.();
+        if (originalEvent && rect) {
+            return {
+                x: (originalEvent.clientX - rect.left) / Math.max(rect.width, 1),
+                y: (originalEvent.clientY - rect.top) / Math.max(rect.height, 1)
+            };
+        }
+
+        const globalPoint = event?.data?.global;
+        const screen = this.app?.screen;
+        if (!globalPoint || !screen) return null;
+        return {
+            x: globalPoint.x / Math.max(screen.width, 1),
+            y: globalPoint.y / Math.max(screen.height, 1)
+        };
+    },
+
+    isPointInZone(x, y, left, top, width, height) {
+        return x >= left && x <= left + width && y >= top && y <= top + height;
+    },
+
+    cycleExpression() {
+        if (!this.model) return;
+        this.expIndex = (this.expIndex + 1) % this.expressionNames.length;
+        const nextExp = this.expressionNames[this.expIndex];
+        
+        console.log(`[Live2D] Triggering expression: ${nextExp.id} (${nextExp.label})`);
+        this.applyExpression(nextExp.id);
+        
+        this.showStatus(nextExp.label);
+        this.showExpressionToast(`Expression: ${nextExp.label}`);
+    },
+
+    showExpressionToast(text, duration = 1600) {
+        const toast = this.expressionToastEl || document.getElementById('expression-toast');
+        if (!toast) return;
+
+        this.expressionToastEl = toast;
+        toast.textContent = text;
+        toast.classList.add('is-visible');
+
+        if (this.expressionToastTimeout) clearTimeout(this.expressionToastTimeout);
+        this.expressionToastTimeout = setTimeout(() => {
+            toast.classList.remove('is-visible');
+        }, duration);
+    },
+
+    applyExpression(expressionId) {
+        if (!this.model) return;
+
+        this.resetExpressionParams();
+        if (!expressionId) {
+            this.clearExpressionState();
+            return;
+        }
+
+        try {
+            this.model.expression(expressionId);
+        } catch (error) {
+            console.error(`[Live2D] Failed to apply expression: ${expressionId}`, error);
+        }
+    },
+
+    resetExpressionParams() {
+        const core = this.model?.internalModel?.coreModel;
+        if (!core) return;
+
+        this.expressionParamIds.forEach(id => {
+            try { core.setParameterValueById(id, 0); } catch (_) {}
+        });
+    },
+
+    clearExpressionState() {
+        const expressionManager =
+            this.model?.internalModel?.motionManager?.expressionManager ||
+            this.model?.internalModel?.expressionManager;
+
+        try {
+            if (expressionManager?.defaultExpression) {
+                expressionManager.currentExpression = expressionManager.defaultExpression;
+                expressionManager.reserveExpressionIndex = -1;
+                expressionManager.resetExpression();
+            } else {
+                this.model.expression(null);
+            }
+        } catch (error) {
+            console.error('[Live2D] Failed to clear expression state:', error);
+        }
+
+        this.resetExpressionParams();
+        requestAnimationFrame(() => this.resetExpressionParams());
+    },
+
+    startLipSync() {
+        if (!this.model || this.lipSyncInterval) return;
+        
+        this.model.motion('Speak', 0).catch(() => {});
+
+        this.lipSyncInterval = setInterval(() => {
+            if (!this.model) return;
+            const value = Math.random() * 0.8;
+            if (this.model.internalModel && this.model.internalModel.coreModel) {
+                const core = this.model.internalModel.coreModel;
+                const mouthIds = ['ParamMouthOpenY', 'ParamMouthOpen', 'PARAM_MOUTH_OPEN_Y'];
+                mouthIds.forEach(id => {
+                    try { core.setParameterValueById(id, value); } catch(e) {}
+                });
+            }
+        }, 80);
+    },
+
+    stopLipSync() {
+        if (this.lipSyncInterval) {
+            clearInterval(this.lipSyncInterval);
+            this.lipSyncInterval = null;
+        }
+        if (this.model) {
+            if (this.model.internalModel && this.model.internalModel.coreModel) {
+                const core = this.model.internalModel.coreModel;
+                const mouthIds = ['ParamMouthOpenY', 'ParamMouthOpen', 'PARAM_MOUTH_OPEN_Y'];
+                mouthIds.forEach(id => {
+                    try { core.setParameterValueById(id, 0); } catch(e) {}
+                });
+            }
+            this.model.motion('Idle', 0).catch(() => {});
+        }
+    }
+};