naisys 1.1.0 → 1.3.0

package/README.md

@@ -11,6 +11,10 @@ vim or nano so point the LLM to use cat to read/write files in a single operatio
 
 [NPM](https://www.npmjs.com/package/naisys) | [Website](https://naisys.org) | [Discord](https://discord.gg/JBUPWSbaEt) | [Demo Video](https://www.youtube.com/watch?v=Ttya3ixjumo)
 
+```bash
+npm install -g naisys
+```
+
 #### Node.js is used to create a simple proxy shell environment for the LLM that
 
 - Helps the LLM keep track of its current context size
@@ -61,10 +65,15 @@ title: Software Engineer
 
 # The model to use for console interactions
 # (gpt4turbo, gpt4turbo, gemini-pro, claude3sonnet, claude3opus, local)
-shellModel: claude3sonnet
+shellModel: gpt4turbo
 
-# The model to use for llmynx, pre-processing websites to fit into a smaller context
-webModel: gpt3turbo
+# Only used between sessions to provide guidance for the next session (use a more powerful model for this)
+# defaults to the shellModel if omitted
+dreamModel: claude3opus
+
+# The model to use for llmynx, pre-processing websites to fit into a smaller context (use a cheaper model)
+# defaults to the shellModel if omitted
+webModel: gemini-pro
 
 # A system like prompt explaining the agent's role and responsibilities
 # You can use config variables in this string
@@ -85,20 +94,26 @@ tokenMax: 5000
 # No value or zero means wait indefinitely (debug driven)
 debugPauseSeconds: 5
 
-# If true, regardless of the debugPauseSeconds, the agent will not wake up on messages
-# With lots of agents this could be costly if they all end up mailing/replying each other in quick succession
+# If true, regardless of the debugPauseSeconds, the agent will wake up on messages
+# Useful for agents with long debugPauseSeconds, so that they can wake up and reply quickly
 wakeOnMessage: false
 
 # The maximum amount to spend on LLM interactions
 # Once reached the agent will stop and this value will need to be increased to continue
 spendLimitDollars: 2.00
 
-# None: Commands from the LLM run automatically, this is the default setting as well if the value is not set
-# Manual: Every command the LLM wants you run you have to approve [y/n]
-# Auto: All commands are run through the separate LLM instace, commands that look like they'll modify the system are blocked
-commandProtection: 'none'
-
-# Additional custom variables can be defined here and/or in the .env file to be loaded into the agent prompt
+# Command Protection: Useful for agents you want to restrict from modifying the system
+#   None: Commands from the LLM run automatically, this is the default setting as well if the value is not set
+#   Manual: Every command the LLM wants to run has to be approved [y/n]. Not very autonomous.
+#   Auto: All commands are run through the separate LLM instace that will check to see if the command is safe
+commandProtection: "none"
+
+# Run these commands on session start, in the example below the agent will see how to use mail and a list of other agents
+initialCommands:
+  - llmail users
+  - llmail help
+  - cat ${env.NAISYS_FOLDER}/home/${agent.username}/PLAN.md
+# Additional custom variables can be defined here and/or in the agent config to be loaded into the agent prompt
 ```
 
 - Run `naisys <path to yaml or directory>`

package/bin/comment

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# ./src/command/commandHandler.ts has the same message
+echo "Comment noted. Try running commands now to achieve your goal. ."

package/bin/endsession

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+echo "'endsession' cannot be used with other commands on the same prompt."

package/bin/llmail

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+echo "'llmail' cannot be used with other commands on the same prompt."

package/bin/llmynx

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+echo "'llmynx' cannot be used with other commands on the same prompt."

package/{naisys.sh → bin/naisys}

@@ -12,7 +12,7 @@ fi
 
 # Resolves the location of naisys from the bin directory
 SCRIPT=$(readlink -f "$0" || echo "$0")
-SCRIPT_DIR=$(dirname "$SCRIPT")
+SCRIPT_DIR=$(dirname "$SCRIPT")/..
 
 # if path is a yaml file then start a single agent
 if [ -f "$1" ]; then

package/bin/pause

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+echo "'pause' cannot be used with other commands on the same prompt."

package/dist/apps/llmail.js

@@ -6,9 +6,8 @@ import * as utilities from "../utils/utilities.js";
 import { naisysToHostPath } from "../utils/utilities.js";
 const _dbFilePath = naisysToHostPath(`${config.naisysFolder}/lib/llmail.db`);
 let _myUserId = -1;
-// Implement maxes so that LLMs actively manage threads, archive, and create new ones
-const _threadTokenMax = config.agent.tokenMax / 2; // So 4000, would be 2000 thread max
-const _messageTokenMax = _threadTokenMax / 5; // Given the above a 400 token max, and 5 big messages per thread
+/** Threading is not currently used so this doesn't matter */
+const _threadTokenMax = config.mailMessageTokenMax * 5;
 /** The 'non-simple' version of this is a thread first mail system. Where agents can create threads, add users, and reply to threads, etc..
  * The problem with this was the agents were too chatty with so many mail commands, wasting context replying, reading threads, etc..
  * Simple mode only has two commands. It still requires db persistance to support offline agents. */
@@ -85,7 +84,9 @@ export async function handleCommand(args) {
             if (simpleMode) {
                 return `llmail <command>
   users: Get list of users on the system
-  send "<users>" "subject" "message": Send a message. ${_messageTokenMax} token max.`;
+  send "<users>" "subject" "message": Send a message. ${config.mailMessageTokenMax} token max.
+  
+* Attachments are not supported, use file paths to refence files in emails as all users are on the same machine`;
             }
             else {
                 return `llmail <command>
@@ -138,7 +139,8 @@ export async function handleCommand(args) {
             await init();
             return "llmail database reset";
         default:
-            return "Unknown llmail command: " + argParams[0];
+            return ("Error, unknown command. See valid commands below:\n" +
+                (await handleCommand("help")));
     }
 }
 export async function getUnreadThreads() {
@@ -339,8 +341,8 @@ async function getUser(db, username) {
 }
 function validateMsgTokenCount(message) {
     const msgTokenCount = utilities.getTokenCount(message);
-    if (msgTokenCount > _messageTokenMax) {
-        throw `Error: Message is ${msgTokenCount} tokens, exceeding the limit of ${_messageTokenMax} tokens`;
+    if (msgTokenCount > config.mailMessageTokenMax) {
+        throw `Error: Message is ${msgTokenCount} tokens, exceeding the limit of ${config.mailMessageTokenMax} tokens`;
     }
     return msgTokenCount;
 }

package/dist/apps/llmynx.js

@@ -8,6 +8,7 @@ import * as output from "../utils/output.js";
 import * as utilities from "../utils/utilities.js";
 // A bad play on words, but this is like lynx but for LLMs..
 let debugMode = false;
+const _contentCache = new Map();
 /** Links numbers are unique in the context so that `llmynx follow <linknum>` can be called on all previous output */
 const _globalLinkMap = new Map();
 const _globalUrlMap = new Map();
@@ -15,63 +16,106 @@ let _nextGlobalLinkNum = 1;
 export async function handleCommand(cmdArgs) {
     outputInDebugMode("LLMYNX DEBUG MODE IS ON");
     const argParams = cmdArgs.split(" ");
-    const defualtTokenMax = config.agent.tokenMax / 8;
     if (!argParams[0]) {
         argParams[0] = "help";
     }
     switch (argParams[0]) {
         case "help":
-            return `llmynx <command> (results will be reduced to around ${defualtTokenMax})
+            return `llmynx <command> (results will be reduced to around ${config.webTokenMax} tokens)
   search <query>: Search google for the given query
   open <url>: Opens the given url. Links are represented as numbers in brackets which prefix the word they are linking like [123]
   follow <link number>: Opens the given link number. Link numbers work across all previous outputs
-  links <url> <page>: Lists only the links for the given url. Use the page number to get more links`;
+  links <url> <page>: Lists only the links for the given url. Use the page number to get more links
+  
+*llmynx does not support input. Use llmynx or curl to call APIs directly*`;
         case "search": {
             const query = argParams.slice(1).join(" ");
-            return await loadUrl("https://www.google.com/search?q=" + encodeURIComponent(query), config.agent.tokenMax / 2, // Prevent form being reduced as google results are usually short anyways and we want to maintainq the links
-            true, true);
+            return await loadUrlContent("https://www.google.com/search?q=" + encodeURIComponent(query), true, true);
         }
         case "open": {
             const url = argParams[1];
-            const isNumber = !isNaN(parseInt(argParams[2]));
-            const tokenMax = isNumber ? parseInt(argParams[2]) : defualtTokenMax;
-            return await loadUrl(url, tokenMax, false, true);
+            return await loadUrlContent(url, false, true);
         }
         case "follow": {
             const linkNum = parseInt(argParams[1]);
-            const isNumber = !isNaN(parseInt(argParams[2]));
-            const tokenMax = isNumber ? parseInt(argParams[2]) : defualtTokenMax;
             const linkUrl = _globalLinkMap.get(linkNum);
             if (!linkUrl) {
                 return "Link number not found";
             }
-            return await loadUrl(linkUrl, tokenMax, true, false);
+            return await loadUrlContent(linkUrl, true, false);
         }
         case "links": {
             const url = argParams[1];
             const isNumber = !isNaN(parseInt(argParams[2]));
             const pageNumber = isNumber ? parseInt(argParams[2]) : 1;
-            return await loadUrl(url, 600, false, false, pageNumber);
+            return await loadUrlLinks(url, pageNumber);
         }
         // Secret command to toggle debug mode
         case "debug":
             debugMode = !debugMode;
             return "Debug mode toggled " + (debugMode ? "on" : "off");
         default:
-            return "Unknown llmynx command: " + argParams[0];
+            return ("Error, unknown command. See valid commands below:\n" +
+                (await handleCommand("help")));
     }
 }
-async function loadUrl(url, tokenMax, showUrl, showFollowHint, linkPageAsContent) {
+/** The content here is not reduced by an LLM, just a paged list of global links is returned */
+async function loadUrlLinks(url, linkPageAsContent) {
     let content = await runLynx(url);
     let links = "";
     // Reverse find 'References: ' and cut everything after it from the content
     const refPos = content.lastIndexOf("References\n");
+    if (refPos > 0) {
+        links = content.slice(refPos);
+        content = "";
+    }
+    else {
+        return "No Links Found";
+    }
+    // Iterate links and de-duplicate
+    const linkLines = links.split("\n");
+    const linkSet = new Set();
+    for (const linkLine of linkLines) {
+        const dotPos = linkLine.indexOf(".");
+        if (dotPos < 0) {
+            continue;
+        }
+        const url = linkLine.substring(dotPos + 1).trim();
+        if (!linkSet.has(url)) {
+            linkSet.add(url);
+            content += url + "\n";
+        }
+    }
+    // Get the token size of the output
+    const linksTokenSize = utilities.getTokenCount(content);
+    outputInDebugMode(`Links Token size: ${linksTokenSize}`);
+    // Reduce content using LLM if it's over the token max
+    if (linksTokenSize > config.webTokenMax) {
+        content = await reduceContent(url, content, linksTokenSize, linkPageAsContent);
+    }
+    else {
+        output.comment(`Link Content is already under ${config.webTokenMax} tokens.`);
+        content = globalizeLinkList(content);
+    }
+    return content;
+}
+async function loadUrlContent(url, showUrl, showFollowHint) {
+    const originalContent = await runLynx(url);
+    let content = originalContent;
+    let links = "";
+    // Reverse find 'References: ' and cut everything after it from the content
+    const refPos = content.lastIndexOf("References\n");
     if (refPos > 0) {
         links = content.slice(refPos);
         content = content.slice(0, refPos);
     }
-    if (linkPageAsContent) {
-        content = links;
+    let usingCachedContent = false;
+    if (_contentCache.has(url)) {
+        const cachedContent = _contentCache.get(url);
+        if (cachedContent.originalContent === originalContent) {
+            content = cachedContent.reducedContent;
+            usingCachedContent = true;
+        }
     }
     // Get the token size of the output
     const contentTokenSize = utilities.getTokenCount(content);
@@ -79,41 +123,18 @@ async function loadUrl(url, tokenMax, showUrl, showFollowHint, linkPageAsContent
     outputInDebugMode(`Content Token size: ${contentTokenSize}\n` +
         `Links Token size: ${linksTokenSize}`);
     // Reduce content using LLM if it's over the token max
-    if (contentTokenSize > tokenMax) {
-        const model = getLLModel(config.agent.webModel);
-        // For example if context is 16k, and max tokens is 2k, 3k with 1.5x overrun
-        // That would be 3k for the current compressed content, 10k for the chunk, and 3k for the output
-        let tokenChunkSize = model.maxTokens - tokenMax * 2 * 1.5;
-        if (linkPageAsContent) {
-            tokenChunkSize = tokenMax;
-        }
-        outputInDebugMode(`Token max chunk size: ${tokenChunkSize}`);
-        const pieceCount = Math.ceil(contentTokenSize / tokenChunkSize);
-        const pieceSize = content.length / pieceCount;
-        let reducedOutput = "";
-        for (let i = 0; i < pieceCount; i++) {
-            const startPos = i * pieceSize;
-            const pieceStr = content.substring(startPos, startPos + pieceSize);
-            if (linkPageAsContent) {
-                if (linkPageAsContent === i + 1) {
-                    return formatLinkPiece(pieceStr);
-                }
-                continue;
-            }
-            output.comment(`Processing Piece ${i + 1} of ${pieceCount}...`);
-            outputInDebugMode(`  Reduced output tokens: ${utilities.getTokenCount(reducedOutput)}\n` +
-                `  Current Piece tokens: ${utilities.getTokenCount(pieceStr)}`);
-            reducedOutput = await llmReduce(url, reducedOutput, i + 1, pieceCount, pieceStr, tokenMax);
-        }
-        if (linkPageAsContent) {
-            return "";
-        }
-        content = reducedOutput;
-        const finalTokenSize = utilities.getTokenCount(reducedOutput);
-        output.comment(`Content reduced from ${contentTokenSize} to ${finalTokenSize} tokens`);
+    if (usingCachedContent) {
+        output.comment("No changes detected, using already cached reduced content");
+    }
+    else if (contentTokenSize > config.webTokenMax) {
+        content = await reduceContent(url, content, contentTokenSize);
+        _contentCache.set(url, {
+            originalContent,
+            reducedContent: content,
+        });
     }
     else {
-        output.comment(`Content is already under ${tokenMax} tokens.`);
+        output.comment(`Content is already under ${config.webTokenMax} tokens.`);
     }
     // Prefix content with url if following as otherwise the url is never shown
     if (showUrl) {
@@ -132,33 +153,70 @@ async function runLynx(url) {
         const modeParams = "";
         const ifWindows = os.platform() === "win32" ? "wsl " : "";
         exec(`${ifWindows}lynx -dump ${modeParams} "${url}"`, (error, stdout, stderr) => {
-            if (error) {
-                resolve(`error: ${error.message}`);
-                return;
+            let output = "";
+            if (stdout) {
+                output += stdout;
+            }
+            // I've only seen either/or, but just in case
+            if (stdout && stderr) {
+                output += "\nError:\n";
             }
             if (stderr) {
-                resolve(`stderr: ${stderr}`);
-                return;
+                output += stderr;
             }
-            resolve(stdout);
+            resolve(output);
         });
     });
 }
-async function llmReduce(url, reducedOutput, pieceNumber, pieceTotal, pieceStr, tokenMax) {
-    const systemMessage = `You will be iteratively fed the web page ${url} broken into ${pieceTotal} sequential equally sized pieces.
-Each piece should be reduced into the final content in order to maintain the meaning of the page while reducing verbosity and duplication.
-The final output should be around ${tokenMax} tokens. 
-Don't remove links which are represented as numbers in brackets which prefix the word they are linking like [123].
+async function reduceContent(url, content, contentTokenSize, linkPageAsContent) {
+    const model = getLLModel(config.agent.webModel);
+    // For example if context is 16k, and max tokens is 2k, 3k with 1.5x overrun
+    // That would be 3k for the current compressed content, 10k for the chunk, and 3k for the output
+    let tokenChunkSize = model.maxTokens - config.webTokenMax * 2 * 1.5;
+    if (linkPageAsContent) {
+        tokenChunkSize = config.webTokenMax;
+    }
+    outputInDebugMode(`Token max chunk size: ${tokenChunkSize}`);
+    const pieceCount = Math.ceil(contentTokenSize / tokenChunkSize);
+    const pieceSize = content.length / pieceCount;
+    let reducedOutput = "";
+    for (let i = 0; i < pieceCount; i++) {
+        const startPos = i * pieceSize;
+        const pieceStr = content.substring(startPos, startPos + pieceSize);
+        if (linkPageAsContent) {
+            if (linkPageAsContent === i + 1) {
+                return globalizeLinkList(pieceStr);
+            }
+            continue;
+        }
+        output.comment(`Processing Piece ${i + 1} of ${pieceCount} with ${model.key}...`);
+        outputInDebugMode(`  Reduced output tokens: ${utilities.getTokenCount(reducedOutput)}\n` +
+            `  Current Piece tokens: ${utilities.getTokenCount(pieceStr)}`);
+        reducedOutput = await llmReduce(url, reducedOutput, i + 1, pieceCount, pieceStr);
+    }
+    if (linkPageAsContent) {
+        return "";
+    }
+    const finalTokenSize = utilities.getTokenCount(reducedOutput);
+    output.comment(`Content reduced from ${contentTokenSize} to ${finalTokenSize} tokens`);
+    return reducedOutput;
+}
+async function llmReduce(url, reducedOutput, pieceNumber, pieceTotal, pieceStr) {
+    const systemMessage = `You will be iteratively fed the web page "${url}" broken into ${pieceTotal} pieces.
+Each 'Web Page Piece' should be merged with the  in order 'Current Reduced Content' to maintain the meaning of the page while reducing verbosity and duplication.
+The final output should be around ${config.webTokenMax} tokens. 
+Links are represented as numbers in brackets, for example [4]. Try not to remove them in the 'Final Merged Content'
 Try to prioritize content of substance over advertising content.`;
-    const content = `Web page piece ${pieceNumber} of ${pieceTotal}: 
+    const content = `Web Page Piece ${pieceNumber} of ${pieceTotal}: 
 ${pieceStr}
 
-Current reduced content: 
+Please merge the 'Web Page Piece' above into the 'Current Reduced Content' below while keeping the result to around ${config.webTokenMax} tokens.
+
+Current Reduced Content: 
 ${reducedOutput}
 
-Please merge the new piece into the existing reduced content above while keeping the result to around ${tokenMax} tokens.
 
-Merged reduced content:
+Final Merged Content:
 `;
     const context = {
         role: LlmRole.User,
@@ -207,23 +265,19 @@ function registerUrl(url) {
     }
     return globalLinkNum;
 }
-function formatLinkPiece(pieceStr) {
+function globalizeLinkList(pieceStr) {
     const alreadySeen = new Set();
     const linkLines = pieceStr.split("\n");
-    let links = "";
+    let globalLinks = "";
     for (const linkLine of linkLines) {
-        const dotPos = linkLine.indexOf(".");
-        if (dotPos < 0) {
-            continue;
-        }
-        const url = linkLine.substring(dotPos + 1).trim();
-        if (alreadySeen.has(url)) {
+        const url = linkLine.trim();
+        if (!url || alreadySeen.has(url)) {
             continue;
         }
         alreadySeen.add(url);
         const globalLinkNum = registerUrl(url);
-        links += `[${globalLinkNum}]${url}\n`;
+        globalLinks += `[${globalLinkNum}]${url}\n`;
     }
-    return links;
+    return globalLinks;
 }
 //# sourceMappingURL=llmynx.js.map

package/dist/command/commandHandler.js

@@ -5,9 +5,9 @@ import * as config from "../config.js";
 import * as contextManager from "../llm/contextManager.js";
 import { ContentSource } from "../llm/contextManager.js";
 import * as costTracker from "../llm/costTracker.js";
+import * as dreamMaker from "../llm/dreamMaker.js";
 import * as inputMode from "../utils/inputMode.js";
 import { InputMode } from "../utils/inputMode.js";
-import * as logService from "../utils/logService.js";
 import * as output from "../utils/output.js";
 import { OutputColor } from "../utils/output.js";
 import * as utilities from "../utils/utilities.js";
@@ -20,7 +20,6 @@ export var NextCommandAction;
     NextCommandAction[NextCommandAction["EndSession"] = 1] = "EndSession";
     NextCommandAction[NextCommandAction["ExitApplication"] = 2] = "ExitApplication";
 })(NextCommandAction || (NextCommandAction = {}));
-export let previousSessionNotes = await logService.getPreviousEndSessionNote();
 export async function processCommand(prompt, consoleInput) {
     // We process the lines one at a time so we can support multiple commands with line breaks
     let firstLine = true;
@@ -61,16 +60,18 @@ export async function processCommand(prompt, consoleInput) {
         switch (cmdParams[0]) {
             case "comment": {
                 // Important - Hint the LLM to turn their thoughts into accounts
+                // ./bin/comment shell script has the same message
                 await contextManager.append("Comment noted. Try running commands now to achieve your goal.");
                 break;
             }
             case "endsession": {
                 // Don't need to check end line as this is the last command in the context, just read to the end
-                previousSessionNotes = utilities.trimChars(cmdArgs, '"');
-                if (!previousSessionNotes) {
+                const endSessionNotes = utilities.trimChars(cmdArgs, '"');
+                if (!endSessionNotes) {
                     await contextManager.append(`End session notes are required. Use endsession "<notes>"`);
                     break;
                 }
+                await dreamMaker.goodnight();
                 await output.commentAndLog("------------------------------------------------------");
                 nextCommandAction = NextCommandAction.EndSession;
                 processNextLLMpromptBlock = false;

package/dist/command/commandLoop.js

@@ -5,6 +5,7 @@ import * as llmynx from "../apps/llmynx.js";
 import * as config from "../config.js";
 import * as contextManager from "../llm/contextManager.js";
 import { ContentSource } from "../llm/contextManager.js";
+import * as dreamMaker from "../llm/dreamMaker.js";
 import { LlmRole } from "../llm/llmDtos.js";
 import * as llmService from "../llm/llmService.js";
 import * as inputMode from "../utils/inputMode.js";
@@ -33,11 +34,13 @@ export async function run() {
     while (nextCommandAction != NextCommandAction.ExitApplication) {
         inputMode.toggle(InputMode.LLM);
         await output.commentAndLog("Starting Context:");
-        await contextManager.append("Previous Session Note:");
-        await contextManager.append(commandHandler.previousSessionNotes || "None");
-        if (await llmail.hasMultipleUsers()) {
-            await commandHandler.processCommand(await promptBuilder.getPrompt(), "llmail help");
-            await commandHandler.processCommand(await promptBuilder.getPrompt(), "llmail users");
+        const latestDream = await dreamMaker.goodmorning();
+        if (latestDream) {
+            await contextManager.append("Previous Session Notes:");
+            await contextManager.append(latestDream);
+        }
+        for (const initialCommand of config.agent.initialCommands) {
+            await commandHandler.processCommand(await promptBuilder.getPrompt(0, false), config.resolveConfigVars(initialCommand));
         }
         inputMode.toggle(InputMode.Debug);
         let pauseSeconds = config.agent.debugPauseSeconds;

package/dist/command/shellCommand.js

@@ -38,7 +38,10 @@ export async function handleCommand(input) {
         if (tokenCount > config.shellOutputTokenMax) {
             outputLimitExceeded = true;
             const trimLength = (text.length * config.shellOutputTokenMax) / tokenCount;
-            text = text.slice(0, trimLength);
+            text =
+                text.slice(0, trimLength / 2) +
+                    "\n\n...\n\n" +
+                    text.slice(-trimLength / 2);
         }
         await contextManager.append(text);
         if (outputLimitExceeded) {

package/dist/command/shellWrapper.js

@@ -170,14 +170,16 @@ function resetProcess() {
  * May also help with common escaping errors */
 function runCommandFromScript(command) {
     const scriptPath = `${config.naisysFolder}/home/${config.agent.username}/.command.tmp.sh`;
-    // set -e causes the script to exit on any error
+    // set -e causes the script to exit on the first error
     const scriptContent = `#!/bin/bash
 set -e
 cd ${_currentPath}
 ${command.trim()}`;
     // create/writewrite file
     fs.writeFileSync(naisysToHostPath(scriptPath), scriptContent);
-    // Source will run the script in the current shell, so any change directories in the script should persist in the current shell
-    return `source ${scriptPath}`;
+    // `Path` is set to the ./bin folder because custom NAISYS commands that follow shell commands will be handled by the shell, which will fail
+    // so we need to remind the LLM that 'naisys commands cannot be used with other commands on the same prompt'
+    // `source` will run the script in the current shell, so any change directories in the script will persist in the current shell
+    return `PATH=${config.binPath}:$PATH source ${scriptPath}`;
 }
 //# sourceMappingURL=shellWrapper.js.map

package/dist/config.js

@@ -8,8 +8,10 @@ program.argument("<agent-path>", "Path to agent configuration file").parse();
 dotenv.config();
 /** The system name that shows after the @ in the command prompt */
 export const hostname = "naisys";
-export const shellOutputTokenMax = 2500;
-export const shellCommmandTimeoutSeconds = 10;
+export const shellOutputTokenMax = 2500; // Limits the size of files that can be read/wrote
+export const shellCommmandTimeoutSeconds = 15; // The number of seconds NAISYS will wait for a shell command to complete
+export const webTokenMax = 2500;
+export const mailMessageTokenMax = 400;
 /* .env is used for global configs across naisys, while agent configs are for the specific agent */
 export const naisysFolder = getEnv("NAISYS_FOLDER", true);
 export const websiteFolder = getEnv("WEBSITE_FOLDER");
@@ -21,31 +23,44 @@ export const anthropicApiKey = getEnv("ANTHROPIC_API_KEY");
 export const agent = loadAgentConfig();
 function loadAgentConfig() {
     const agentPath = program.args[0];
-    const checkAgentConfig = yaml.load(fs.readFileSync(agentPath, "utf8"));
+    const config = yaml.load(fs.readFileSync(agentPath, "utf8"));
     // throw if any property is undefined
     for (const key of [
         "username",
         "title",
         "shellModel",
-        "webModel",
         "agentPrompt",
         "spendLimitDollars",
         "tokenMax",
         // other properties can be undefined
     ]) {
-        if (!valueFromString(checkAgentConfig, key)) {
+        if (!valueFromString(config, key)) {
             throw `Agent config: Error, ${key} is not defined`;
         }
     }
-    if (!checkAgentConfig.commandProtection) {
-        checkAgentConfig.commandProtection = CommandProtection.None;
+    // Sanitize input
+    if (!config.initialCommands) {
+        config.initialCommands = [];
     }
-    if (!Object.values(CommandProtection).includes(checkAgentConfig.commandProtection)) {
+    else if (!Array.isArray(config.initialCommands)) {
+        throw `Agent config: Error, 'initialCommands' is not an array`;
+    }
+    config.debugPauseSeconds = config.debugPauseSeconds
+        ? Number(config.debugPauseSeconds)
+        : 0;
+    config.wakeOnMessage = Boolean(config.wakeOnMessage);
+    config.webModel || (config.webModel = config.shellModel);
+    config.dreamModel || (config.dreamModel = config.shellModel);
+    if (!config.commandProtection) {
+        config.commandProtection = CommandProtection.None;
+    }
+    if (!Object.values(CommandProtection).includes(config.commandProtection)) {
         throw `Agent config: Error, 'commandProtection' is not a valid value`;
     }
-    return checkAgentConfig;
+    return config;
 }
 export const packageVersion = await getVersion();
+export const binPath = getBinPath();
 /** Can only get version from env variable when naisys is started with npm,
  * otherwise need to rip it from the package ourselves relative to where this file is located */
 async function getVersion() {
@@ -67,4 +82,28 @@ function getEnv(key, required) {
     }
     return value;
 }
+export function resolveConfigVars(templateString) {
+    let resolvedString = templateString;
+    resolvedString = resolveTemplateVars(resolvedString, "agent", agent);
+    resolvedString = resolveTemplateVars(resolvedString, "env", process.env);
+    return resolvedString;
+}
+function resolveTemplateVars(templateString, allowedVarString, mappedVar) {
+    const pattern = new RegExp(`\\$\\{${allowedVarString}\\.([^}]+)\\}`, "g");
+    return templateString.replace(pattern, (match, key) => {
+        const value = valueFromString(mappedVar, key);
+        if (value === undefined) {
+            throw `Agent config: Error, ${key} is not defined`;
+        }
+        return value;
+    });
+}
+function getBinPath() {
+    // C:/git/naisys/dist/config.js
+    let binPath = new URL("../bin", import.meta.url).pathname;
+    if (binPath.startsWith("/C:")) {
+        binPath = "/mnt/c" + binPath.substring(3);
+    }
+    return binPath;
+}
 //# sourceMappingURL=config.js.map

package/dist/llm/contextManager.js

@@ -5,7 +5,6 @@ import * as logService from "../utils/logService.js";
 import * as output from "../utils/output.js";
 import { OutputColor } from "../utils/output.js";
 import * as utilities from "../utils/utilities.js";
-import { valueFromString } from "../utils/utilities.js";
 import { LlmRole } from "./llmDtos.js";
 export var ContentSource;
 (function (ContentSource) {
@@ -23,8 +22,7 @@ export function getSystemMessage() {
     // A lot of the stipulations in here are to prevent common LLM mistakes
     // Like we can't jump between standard and special commands in a single prompt, which the LLM will try to do if not warned
     let agentPrompt = config.agent.agentPrompt;
-    agentPrompt = resolveTemplateVars(agentPrompt, "agent", config.agent);
-    agentPrompt = resolveTemplateVars(agentPrompt, "env", process.env);
+    agentPrompt = config.resolveConfigVars(agentPrompt);
     const systemMessage = `${agentPrompt.trim()}
 
 This is a command line interface presenting you with the next command prompt. 
@@ -38,12 +36,12 @@ NAISYS ${config.packageVersion} Shell
 Welcome back ${config.agent.username}!
 MOTD:
 Date: ${new Date().toLocaleString()}
-Commands: 
-  Standard Unix commands are available
+LINUX Commands: 
+  Standard Linux commands are available
   vi and nano are not supported
   Read files with cat. Write files with \`cat > filename << 'EOF'\`
   Do not input notes after the prompt. Only valid commands.
-Special Commands: (Don't mix with standard commands on the same prompt)
+NAISYS Commands: (cannot be used with other commands on the same prompt)
   llmail: A local mail system for communicating with your team
   llmynx: A context optimized web browser. Enter 'llmynx help' to learn how to use it
   comment "<thought>": Any non-command output like thinking out loud, prefix with the 'comment' command
@@ -51,23 +49,12 @@ Special Commands: (Don't mix with standard commands on the same prompt)
   endsession "<note>": Ends this session, clears the console log and context.
     The note should help you find your bearings in the next session. 
     The note should contain your next goal, and important things should you remember.
-    Try to keep the note around 400 tokens.
 Tokens:
   The console log can only hold a certain number of 'tokens' that is specified in the prompt
   Make sure to call endsession before the limit is hit so you can continue your work with a fresh console`;
     _cachedSystemMessage = systemMessage;
     return systemMessage;
 }
-function resolveTemplateVars(templateString, allowedVarString, mappedVar) {
-    const pattern = new RegExp(`\\$\\{${allowedVarString}\\.([^}]+)\\}`, "g");
-    return templateString.replace(pattern, (match, key) => {
-        const value = valueFromString(mappedVar, key);
-        if (value === undefined) {
-            throw `Agent config: Error, ${key} is not defined`;
-        }
-        return value;
-    });
-}
 export let messages = [];
 export async function append(text, source = ContentSource.Console) {
     // Debug runs in a shadow mode where their activity is not recorded in the context

package/dist/llm/dreamMaker.js

@@ -0,0 +1,71 @@
+import * as config from "../config.js";
+import * as dbUtils from "../utils/dbUtils.js";
+import * as output from "../utils/output.js";
+import { naisysToHostPath } from "../utils/utilities.js";
+import * as contextManager from "./contextManager.js";
+import { LlmRole } from "./llmDtos.js";
+import * as llmService from "./llmService.js";
+const _dbFilePath = naisysToHostPath(`${config.naisysFolder}/lib/dream.db`);
+await init();
+async function init() {
+    const newDbCreated = await dbUtils.initDatabase(_dbFilePath);
+    await usingDatabase(async (db) => {
+        if (!newDbCreated) {
+            return;
+        }
+        await db.exec(`CREATE TABLE DreamLog (
+      id INTEGER PRIMARY KEY, 
+      username TEXT NOT NULL,
+      date TEXT NOT NULL,
+      dream TEXT NOT NULL
+    )`);
+    });
+}
+export async function goodmorning() {
+    return await usingDatabase(async (db) => {
+        const row = await db.get(`SELECT dream 
+       FROM DreamLog 
+       WHERE username = ? 
+       ORDER BY date DESC LIMIT 1`, config.agent.username);
+        return row === null || row === void 0 ? void 0 : row.dream;
+    });
+}
+export async function goodnight() {
+    output.comment("Dreaming about the session...");
+    const dream = await runDreamSequence();
+    await storeDream(dream);
+    return dream;
+}
+async function runDreamSequence() {
+    const systemMessage = `You are ${config.agent.username}'s unconcious sleep process. You compile all ${config.agent.username}'s
+thoughts during the day and reduce them down to important things to remember - references, plans, project structure, schemas, 
+file locations, urls, and more. You are the sleep process, and you are the most important process. Using your results, 
+when ${config.agent.username} wakes up they'll know exactly what to do and how to do it.`;
+    const allTheThings = contextManager.messages.map((m) => m.content).join("\n");
+    return await llmService.query(config.agent.dreamModel, systemMessage, [
+        {
+            role: LlmRole.User,
+            content: allTheThings,
+        },
+        {
+            role: LlmRole.Assistant,
+            content: "We sure had an eventful day",
+        },
+        {
+            role: LlmRole.User,
+            content: `Dream on all these things and let me know what you come up with. Use what was done in the previous session as a guide 
+          for what's possible tomorrow. Don't overload yourself with too many thoughts and ideas. Keep important references for the future
+          but don't go into any great detail of future plans unless it's happening soon. `,
+        },
+    ], "dream");
+}
+async function usingDatabase(run) {
+    return dbUtils.usingDatabase(_dbFilePath, run);
+}
+async function storeDream(dream) {
+    await usingDatabase(async (db) => {
+        await db.run(`INSERT INTO DreamLog (username, date, dream) 
+       VALUES (?, datetime('now'), ?)`, config.agent.username, dream);
+    });
+}
+//# sourceMappingURL=dreamMaker.js.map

package/dist/llm/llmService.js

@@ -80,24 +80,37 @@ async function sendWithGoogle(modelKey, systemMessage, context, source) {
     if (lastMessage.role !== LlmRole.User) {
         throw "Error, last message on context is not a user message";
     }
+    const contextHistory = context
+        .filter((m) => m != lastMessage)
+        .map((m) => ({
+        role: m.role == LlmRole.Assistant ? "model" : "user",
+        parts: [
+            {
+                text: m.content,
+            },
+        ],
+    }));
     const history = [
         {
             role: LlmRole.User, // System role is not supported by Google API
-            parts: systemMessage,
+            parts: [
+                {
+                    text: systemMessage,
+                },
+            ],
         },
         {
             role: "model",
-            parts: "Understood",
+            parts: [
+                {
+                    text: "Understood",
+                },
+            ],
         },
-        ...context
-            .filter((m) => m != lastMessage)
-            .map((m) => ({
-            role: m.role == LlmRole.Assistant ? "model" : LlmRole.User,
-            parts: m.content,
-        })),
+        ...contextHistory,
     ];
     const chat = googleModel.startChat({
-        history: history,
+        history,
         generationConfig: {
             maxOutputTokens: 2000,
         },

package/dist/utils/logService.js

@@ -87,19 +87,6 @@ function appendToLogFile(filepath, message) {
       </td>
     </tr>`);
 }
-export function getPreviousEndSessionNote() {
-    // Find the most recent message in the log that starts with 'endsession' for the local user
-    return usingDatabase(async (db) => {
-        const result = await db.get(`SELECT message 
-        FROM ContextLog 
-        WHERE username = ? AND message LIKE 'endsession %' 
-        ORDER BY id DESC 
-        LIMIT 1`, [config.agent.username]);
-        const endSessionMsg = result === null || result === void 0 ? void 0 : result.message;
-        // Trim endsession prefix
-        return (endSessionMsg === null || endSessionMsg === void 0 ? void 0 : endSessionMsg.slice("endsession ".length)) || "";
-    });
-}
 async function usingDatabase(run) {
     return dbUtils.usingDatabase(_dbFilePath, run);
 }

package/package.json

@@ -1,19 +1,19 @@
 {
   "name": "naisys",
   "description": "Node.js Autonomous Intelligence System",
-  "version": "1.1.0",
+  "version": "1.3.0",
   "type": "module",
   "main": "dist/naisys.js",
   "preferGlobal": true,
   "bin": {
-    "naisys": "naisys.sh"
+    "naisys": "bin/naisys"
   },
   "scripts": {
     "compile/run/attachable": "tsc && node --inspect dist/naisys.js ./agents/webdev-fansite.yaml",
-    "run agent:dev": "node dist/naisys.js ./agents/2-team/dev.yaml",
-    "run agent:admin": "node dist/naisys.js ./agents/2-team/admin.yaml",
+    "agent:dev": "node dist/naisys.js ./agents/3-team-dev-db-content/dev.yaml",
+    "agent:db": "node dist/naisys.js ./agents/3-team-dev-db-content/db.yaml",
+    "agent:content": "node dist/naisys.js ./agents/3-team-dev-db-content/content.yaml",
     "clean": "rm -rf dist",
-    "clean:win": "wsl rm -rf dist",
     "compile": "tsc",
     "eslint": "npx eslint --rulesdir eslint-rules src",
     "test": "tsc && node --experimental-vm-modules node_modules/jest/bin/jest.js --testPathPattern=dist/__tests__",
@@ -21,8 +21,9 @@
     "dependency-graph": "madge --image dependency-graph.png dist",
     "detect-cycles": "madge --circular dist",
     "updates:check": "npm-check-updates",
-    "updates:apply": "npm-check-updates -u && npm update",
-    "npm:publish:dryrun": "npm run clean && npm run compile && npm publish --dry-run"
+    "updates:apply": "npm-check-updates -u && npm install",
+    "npm:publish:dryrun": "npm run clean && npm install && npm run compile && npm publish --dry-run",
+    "postinstall": "chmod +x ./bin/*"
   },
   "repository": {
     "type": "git",
@@ -42,25 +43,25 @@
   "devDependencies": {
     "@types/escape-html": "1.0.4",
     "@types/js-yaml": "4.0.9",
-    "@types/node": "20.11.25",
+    "@types/node": "20.11.28",
     "@types/text-table": "0.2.5",
-    "@typescript-eslint/eslint-plugin": "7.1.1",
-    "@typescript-eslint/parser": "7.1.1",
+    "@typescript-eslint/eslint-plugin": "7.2.0",
+    "@typescript-eslint/parser": "7.2.0",
     "eslint": "8.57.0",
     "jest": "29.7.0",
     "prettier": "3.2.5",
     "ts-node": "10.9.2",
-    "typescript": "5.3.3"
+    "typescript": "5.4.2"
   },
   "dependencies": {
-    "@anthropic-ai/sdk": "0.16.1",
-    "@google/generative-ai": "0.2.1",
+    "@anthropic-ai/sdk": "0.18.0",
+    "@google/generative-ai": "0.3.0",
     "chalk": "5.3.0",
     "commander": "12.0.0",
     "dotenv": "16.4.5",
     "escape-html": "1.0.3",
     "js-yaml": "4.1.0",
-    "openai": "4.28.4",
+    "openai": "4.29.0",
     "sqlite": "5.1.1",
     "sqlite3": "5.1.7",
     "text-table": "0.2.0",