naisys 1.2.0 → 1.3.0

package/README.md

@@ -65,10 +65,15 @@ title: Software Engineer
 
 # The model to use for console interactions
 # (gpt4turbo, gpt4turbo, gemini-pro, claude3sonnet, claude3opus, local)
-shellModel: claude3sonnet
+shellModel: gpt4turbo
 
-# The model to use for llmynx, pre-processing websites to fit into a smaller context
-webModel: gpt3turbo
+# Only used between sessions to provide guidance for the next session (use a more powerful model for this)
+# defaults to the shellModel if omitted
+dreamModel: claude3opus
+
+# The model to use for llmynx, pre-processing websites to fit into a smaller context (use a cheaper model)
+# defaults to the shellModel if omitted
+webModel: gemini-pro
 
 # A system like prompt explaining the agent's role and responsibilities
 # You can use config variables in this string

package/dist/apps/llmail.js

@@ -84,7 +84,9 @@ export async function handleCommand(args) {
             if (simpleMode) {
                 return `llmail <command>
   users: Get list of users on the system
-  send "<users>" "subject" "message": Send a message. ${config.mailMessageTokenMax} token max.`;
+  send "<users>" "subject" "message": Send a message. ${config.mailMessageTokenMax} token max.
+  
+* Attachments are not supported, use file paths to refence files in emails as all users are on the same machine`;
             }
             else {
                 return `llmail <command>

package/dist/apps/llmynx.js

@@ -8,6 +8,7 @@ import * as output from "../utils/output.js";
 import * as utilities from "../utils/utilities.js";
 // A bad play on words, but this is like lynx but for LLMs..
 let debugMode = false;
+const _contentCache = new Map();
 /** Links numbers are unique in the context so that `llmynx follow <linknum>` can be called on all previous output */
 const _globalLinkMap = new Map();
 const _globalUrlMap = new Map();
@@ -20,7 +21,7 @@ export async function handleCommand(cmdArgs) {
     }
     switch (argParams[0]) {
         case "help":
-            return `llmynx <command> (results will be reduced to around ${config.webTokenMax})
+            return `llmynx <command> (results will be reduced to around ${config.webTokenMax} tokens)
   search <query>: Search google for the given query
   open <url>: Opens the given url. Links are represented as numbers in brackets which prefix the word they are linking like [123]
   follow <link number>: Opens the given link number. Link numbers work across all previous outputs
@@ -29,11 +30,11 @@ export async function handleCommand(cmdArgs) {
 *llmynx does not support input. Use llmynx or curl to call APIs directly*`;
         case "search": {
             const query = argParams.slice(1).join(" ");
-            return await loadUrl("https://www.google.com/search?q=" + encodeURIComponent(query), true, true);
+            return await loadUrlContent("https://www.google.com/search?q=" + encodeURIComponent(query), true, true);
         }
         case "open": {
             const url = argParams[1];
-            return await loadUrl(url, false, true);
+            return await loadUrlContent(url, false, true);
         }
         case "follow": {
             const linkNum = parseInt(argParams[1]);
@@ -41,13 +42,13 @@ export async function handleCommand(cmdArgs) {
             if (!linkUrl) {
                 return "Link number not found";
             }
-            return await loadUrl(linkUrl, true, false);
+            return await loadUrlContent(linkUrl, true, false);
         }
         case "links": {
             const url = argParams[1];
             const isNumber = !isNaN(parseInt(argParams[2]));
             const pageNumber = isNumber ? parseInt(argParams[2]) : 1;
-            return await loadUrl(url, false, false, pageNumber);
+            return await loadUrlLinks(url, pageNumber);
         }
         // Secret command to toggle debug mode
         case "debug":
@@ -58,17 +59,63 @@ export async function handleCommand(cmdArgs) {
                 (await handleCommand("help")));
     }
 }
-async function loadUrl(url, showUrl, showFollowHint, linkPageAsContent) {
+/** The content here is not reduced by an LLM, just a paged list of global links is returned */
+async function loadUrlLinks(url, linkPageAsContent) {
     let content = await runLynx(url);
     let links = "";
     // Reverse find 'References: ' and cut everything after it from the content
     const refPos = content.lastIndexOf("References\n");
+    if (refPos > 0) {
+        links = content.slice(refPos);
+        content = "";
+    }
+    else {
+        return "No Links Found";
+    }
+    // Iterate links and de-duplicate
+    const linkLines = links.split("\n");
+    const linkSet = new Set();
+    for (const linkLine of linkLines) {
+        const dotPos = linkLine.indexOf(".");
+        if (dotPos < 0) {
+            continue;
+        }
+        const url = linkLine.substring(dotPos + 1).trim();
+        if (!linkSet.has(url)) {
+            linkSet.add(url);
+            content += url + "\n";
+        }
+    }
+    // Get the token size of the output
+    const linksTokenSize = utilities.getTokenCount(content);
+    outputInDebugMode(`Links Token size: ${linksTokenSize}`);
+    // Reduce content using LLM if it's over the token max
+    if (linksTokenSize > config.webTokenMax) {
+        content = await reduceContent(url, content, linksTokenSize, linkPageAsContent);
+    }
+    else {
+        output.comment(`Link Content is already under ${config.webTokenMax} tokens.`);
+        content = globalizeLinkList(content);
+    }
+    return content;
+}
+async function loadUrlContent(url, showUrl, showFollowHint) {
+    const originalContent = await runLynx(url);
+    let content = originalContent;
+    let links = "";
+    // Reverse find 'References: ' and cut everything after it from the content
+    const refPos = content.lastIndexOf("References\n");
     if (refPos > 0) {
         links = content.slice(refPos);
         content = content.slice(0, refPos);
     }
-    if (linkPageAsContent) {
-        content = links;
+    let usingCachedContent = false;
+    if (_contentCache.has(url)) {
+        const cachedContent = _contentCache.get(url);
+        if (cachedContent.originalContent === originalContent) {
+            content = cachedContent.reducedContent;
+            usingCachedContent = true;
+        }
     }
     // Get the token size of the output
     const contentTokenSize = utilities.getTokenCount(content);
@@ -76,38 +123,15 @@ async function loadUrl(url, showUrl, showFollowHint, linkPageAsContent) {
     outputInDebugMode(`Content Token size: ${contentTokenSize}\n` +
         `Links Token size: ${linksTokenSize}`);
     // Reduce content using LLM if it's over the token max
-    if (contentTokenSize > config.webTokenMax) {
-        const model = getLLModel(config.agent.webModel);
-        // For example if context is 16k, and max tokens is 2k, 3k with 1.5x overrun
-        // That would be 3k for the current compressed content, 10k for the chunk, and 3k for the output
-        let tokenChunkSize = model.maxTokens - config.webTokenMax * 2 * 1.5;
-        if (linkPageAsContent) {
-            tokenChunkSize = config.webTokenMax;
-        }
-        outputInDebugMode(`Token max chunk size: ${tokenChunkSize}`);
-        const pieceCount = Math.ceil(contentTokenSize / tokenChunkSize);
-        const pieceSize = content.length / pieceCount;
-        let reducedOutput = "";
-        for (let i = 0; i < pieceCount; i++) {
-            const startPos = i * pieceSize;
-            const pieceStr = content.substring(startPos, startPos + pieceSize);
-            if (linkPageAsContent) {
-                if (linkPageAsContent === i + 1) {
-                    return formatLinkPiece(pieceStr);
-                }
-                continue;
-            }
-            output.comment(`Processing Piece ${i + 1} of ${pieceCount} with ${model.key}...`);
-            outputInDebugMode(`  Reduced output tokens: ${utilities.getTokenCount(reducedOutput)}\n` +
-                `  Current Piece tokens: ${utilities.getTokenCount(pieceStr)}`);
-            reducedOutput = await llmReduce(url, reducedOutput, i + 1, pieceCount, pieceStr);
-        }
-        if (linkPageAsContent) {
-            return "";
-        }
-        content = reducedOutput;
-        const finalTokenSize = utilities.getTokenCount(reducedOutput);
-        output.comment(`Content reduced from ${contentTokenSize} to ${finalTokenSize} tokens`);
+    if (usingCachedContent) {
+        output.comment("No changes detected, using already cached reduced content");
+    }
+    else if (contentTokenSize > config.webTokenMax) {
+        content = await reduceContent(url, content, contentTokenSize);
+        _contentCache.set(url, {
+            originalContent,
+            reducedContent: content,
+        });
     }
     else {
         output.comment(`Content is already under ${config.webTokenMax} tokens.`);
@@ -144,21 +168,55 @@ async function runLynx(url) {
         });
     });
 }
+async function reduceContent(url, content, contentTokenSize, linkPageAsContent) {
+    const model = getLLModel(config.agent.webModel);
+    // For example if context is 16k, and max tokens is 2k, 3k with 1.5x overrun
+    // That would be 3k for the current compressed content, 10k for the chunk, and 3k for the output
+    let tokenChunkSize = model.maxTokens - config.webTokenMax * 2 * 1.5;
+    if (linkPageAsContent) {
+        tokenChunkSize = config.webTokenMax;
+    }
+    outputInDebugMode(`Token max chunk size: ${tokenChunkSize}`);
+    const pieceCount = Math.ceil(contentTokenSize / tokenChunkSize);
+    const pieceSize = content.length / pieceCount;
+    let reducedOutput = "";
+    for (let i = 0; i < pieceCount; i++) {
+        const startPos = i * pieceSize;
+        const pieceStr = content.substring(startPos, startPos + pieceSize);
+        if (linkPageAsContent) {
+            if (linkPageAsContent === i + 1) {
+                return globalizeLinkList(pieceStr);
+            }
+            continue;
+        }
+        output.comment(`Processing Piece ${i + 1} of ${pieceCount} with ${model.key}...`);
+        outputInDebugMode(`  Reduced output tokens: ${utilities.getTokenCount(reducedOutput)}\n` +
+            `  Current Piece tokens: ${utilities.getTokenCount(pieceStr)}`);
+        reducedOutput = await llmReduce(url, reducedOutput, i + 1, pieceCount, pieceStr);
+    }
+    if (linkPageAsContent) {
+        return "";
+    }
+    const finalTokenSize = utilities.getTokenCount(reducedOutput);
+    output.comment(`Content reduced from ${contentTokenSize} to ${finalTokenSize} tokens`);
+    return reducedOutput;
+}
 async function llmReduce(url, reducedOutput, pieceNumber, pieceTotal, pieceStr) {
-    const systemMessage = `You will be iteratively fed the web page ${url} broken into ${pieceTotal} sequential equally sized pieces.
-Each piece should be reduced into the final content in order to maintain the meaning of the page while reducing verbosity and duplication.
+    const systemMessage = `You will be iteratively fed the web page "${url}" broken into ${pieceTotal} pieces.
+Each 'Web Page Piece' should be merged with the  in order 'Current Reduced Content' to maintain the meaning of the page while reducing verbosity and duplication.
 The final output should be around ${config.webTokenMax} tokens. 
-Don't remove links which are represented as numbers in brackets which prefix the word they are linking like [123].
+Links are represented as numbers in brackets, for example [4]. Try not to remove them in the 'Final Merged Content'
 Try to prioritize content of substance over advertising content.`;
-    const content = `Web page piece ${pieceNumber} of ${pieceTotal}: 
+    const content = `Web Page Piece ${pieceNumber} of ${pieceTotal}: 
 ${pieceStr}
 
-Current reduced content: 
+Please merge the 'Web Page Piece' above into the 'Current Reduced Content' below while keeping the result to around ${config.webTokenMax} tokens.
+
+Current Reduced Content: 
 ${reducedOutput}
 
-Please merge the new piece into the existing reduced content above while keeping the result to around ${config.webTokenMax} tokens.
 
-Merged reduced content:
+Final Merged Content:
 `;
     const context = {
         role: LlmRole.User,
@@ -207,23 +265,19 @@ function registerUrl(url) {
     }
     return globalLinkNum;
 }
-function formatLinkPiece(pieceStr) {
+function globalizeLinkList(pieceStr) {
     const alreadySeen = new Set();
     const linkLines = pieceStr.split("\n");
-    let links = "";
+    let globalLinks = "";
     for (const linkLine of linkLines) {
-        const dotPos = linkLine.indexOf(".");
-        if (dotPos < 0) {
-            continue;
-        }
-        const url = linkLine.substring(dotPos + 1).trim();
-        if (alreadySeen.has(url)) {
+        const url = linkLine.trim();
+        if (!url || alreadySeen.has(url)) {
             continue;
         }
         alreadySeen.add(url);
         const globalLinkNum = registerUrl(url);
-        links += `[${globalLinkNum}]${url}\n`;
+        globalLinks += `[${globalLinkNum}]${url}\n`;
     }
-    return links;
+    return globalLinks;
 }
 //# sourceMappingURL=llmynx.js.map

package/dist/command/commandHandler.js

@@ -5,9 +5,9 @@ import * as config from "../config.js";
 import * as contextManager from "../llm/contextManager.js";
 import { ContentSource } from "../llm/contextManager.js";
 import * as costTracker from "../llm/costTracker.js";
+import * as dreamMaker from "../llm/dreamMaker.js";
 import * as inputMode from "../utils/inputMode.js";
 import { InputMode } from "../utils/inputMode.js";
-import * as logService from "../utils/logService.js";
 import * as output from "../utils/output.js";
 import { OutputColor } from "../utils/output.js";
 import * as utilities from "../utils/utilities.js";
@@ -20,7 +20,6 @@ export var NextCommandAction;
     NextCommandAction[NextCommandAction["EndSession"] = 1] = "EndSession";
     NextCommandAction[NextCommandAction["ExitApplication"] = 2] = "ExitApplication";
 })(NextCommandAction || (NextCommandAction = {}));
-export let previousSessionNotes = await logService.getPreviousEndSessionNote();
 export async function processCommand(prompt, consoleInput) {
     // We process the lines one at a time so we can support multiple commands with line breaks
     let firstLine = true;
@@ -67,11 +66,12 @@ export async function processCommand(prompt, consoleInput) {
             }
             case "endsession": {
                 // Don't need to check end line as this is the last command in the context, just read to the end
-                previousSessionNotes = utilities.trimChars(cmdArgs, '"');
-                if (!previousSessionNotes) {
+                const endSessionNotes = utilities.trimChars(cmdArgs, '"');
+                if (!endSessionNotes) {
                     await contextManager.append(`End session notes are required. Use endsession "<notes>"`);
                     break;
                 }
+                await dreamMaker.goodnight();
                 await output.commentAndLog("------------------------------------------------------");
                 nextCommandAction = NextCommandAction.EndSession;
                 processNextLLMpromptBlock = false;

package/dist/command/commandLoop.js

@@ -5,6 +5,7 @@ import * as llmynx from "../apps/llmynx.js";
 import * as config from "../config.js";
 import * as contextManager from "../llm/contextManager.js";
 import { ContentSource } from "../llm/contextManager.js";
+import * as dreamMaker from "../llm/dreamMaker.js";
 import { LlmRole } from "../llm/llmDtos.js";
 import * as llmService from "../llm/llmService.js";
 import * as inputMode from "../utils/inputMode.js";
@@ -33,8 +34,11 @@ export async function run() {
     while (nextCommandAction != NextCommandAction.ExitApplication) {
         inputMode.toggle(InputMode.LLM);
         await output.commentAndLog("Starting Context:");
-        await contextManager.append("Previous Session Note:");
-        await contextManager.append(commandHandler.previousSessionNotes || "None");
+        const latestDream = await dreamMaker.goodmorning();
+        if (latestDream) {
+            await contextManager.append("Previous Session Notes:");
+            await contextManager.append(latestDream);
+        }
         for (const initialCommand of config.agent.initialCommands) {
             await commandHandler.processCommand(await promptBuilder.getPrompt(0, false), config.resolveConfigVars(initialCommand));
         }

package/dist/config.js

@@ -29,7 +29,6 @@ function loadAgentConfig() {
         "username",
         "title",
         "shellModel",
-        "webModel",
         "agentPrompt",
         "spendLimitDollars",
         "tokenMax",
@@ -50,6 +49,8 @@ function loadAgentConfig() {
         ? Number(config.debugPauseSeconds)
         : 0;
     config.wakeOnMessage = Boolean(config.wakeOnMessage);
+    config.webModel || (config.webModel = config.shellModel);
+    config.dreamModel || (config.dreamModel = config.shellModel);
     if (!config.commandProtection) {
         config.commandProtection = CommandProtection.None;
     }

package/dist/llm/dreamMaker.js

@@ -0,0 +1,71 @@
+import * as config from "../config.js";
+import * as dbUtils from "../utils/dbUtils.js";
+import * as output from "../utils/output.js";
+import { naisysToHostPath } from "../utils/utilities.js";
+import * as contextManager from "./contextManager.js";
+import { LlmRole } from "./llmDtos.js";
+import * as llmService from "./llmService.js";
+const _dbFilePath = naisysToHostPath(`${config.naisysFolder}/lib/dream.db`);
+await init();
+async function init() {
+    const newDbCreated = await dbUtils.initDatabase(_dbFilePath);
+    await usingDatabase(async (db) => {
+        if (!newDbCreated) {
+            return;
+        }
+        await db.exec(`CREATE TABLE DreamLog (
+      id INTEGER PRIMARY KEY, 
+      username TEXT NOT NULL,
+      date TEXT NOT NULL,
+      dream TEXT NOT NULL
+    )`);
+    });
+}
+export async function goodmorning() {
+    return await usingDatabase(async (db) => {
+        const row = await db.get(`SELECT dream 
+       FROM DreamLog 
+       WHERE username = ? 
+       ORDER BY date DESC LIMIT 1`, config.agent.username);
+        return row === null || row === void 0 ? void 0 : row.dream;
+    });
+}
+export async function goodnight() {
+    output.comment("Dreaming about the session...");
+    const dream = await runDreamSequence();
+    await storeDream(dream);
+    return dream;
+}
+async function runDreamSequence() {
+    const systemMessage = `You are ${config.agent.username}'s unconcious sleep process. You compile all ${config.agent.username}'s
+thoughts during the day and reduce them down to important things to remember - references, plans, project structure, schemas, 
+file locations, urls, and more. You are the sleep process, and you are the most important process. Using your results, 
+when ${config.agent.username} wakes up they'll know exactly what to do and how to do it.`;
+    const allTheThings = contextManager.messages.map((m) => m.content).join("\n");
+    return await llmService.query(config.agent.dreamModel, systemMessage, [
+        {
+            role: LlmRole.User,
+            content: allTheThings,
+        },
+        {
+            role: LlmRole.Assistant,
+            content: "We sure had an eventful day",
+        },
+        {
+            role: LlmRole.User,
+            content: `Dream on all these things and let me know what you come up with. Use what was done in the previous session as a guide 
+          for what's possible tomorrow. Don't overload yourself with too many thoughts and ideas. Keep important references for the future
+          but don't go into any great detail of future plans unless it's happening soon. `,
+        },
+    ], "dream");
+}
+async function usingDatabase(run) {
+    return dbUtils.usingDatabase(_dbFilePath, run);
+}
+async function storeDream(dream) {
+    await usingDatabase(async (db) => {
+        await db.run(`INSERT INTO DreamLog (username, date, dream) 
+       VALUES (?, datetime('now'), ?)`, config.agent.username, dream);
+    });
+}
+//# sourceMappingURL=dreamMaker.js.map

package/dist/utils/logService.js

@@ -87,19 +87,6 @@ function appendToLogFile(filepath, message) {
       </td>
     </tr>`);
 }
-export function getPreviousEndSessionNote() {
-    // Find the most recent message in the log that starts with 'endsession' for the local user
-    return usingDatabase(async (db) => {
-        const result = await db.get(`SELECT message 
-        FROM ContextLog 
-        WHERE username = ? AND message LIKE 'endsession %' 
-        ORDER BY id DESC 
-        LIMIT 1`, [config.agent.username]);
-        const endSessionMsg = result === null || result === void 0 ? void 0 : result.message;
-        // Trim endsession prefix
-        return (endSessionMsg === null || endSessionMsg === void 0 ? void 0 : endSessionMsg.slice("endsession ".length)) || "";
-    });
-}
 async function usingDatabase(run) {
     return dbUtils.usingDatabase(_dbFilePath, run);
 }

package/package.json

@@ -1,27 +1,28 @@
 {
   "name": "naisys",
   "description": "Node.js Autonomous Intelligence System",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "type": "module",
   "main": "dist/naisys.js",
   "preferGlobal": true,
   "bin": {
-    "naisys": "./bin/naisys"
+    "naisys": "bin/naisys"
   },
   "scripts": {
     "compile/run/attachable": "tsc && node --inspect dist/naisys.js ./agents/webdev-fansite.yaml",
-    "run agent:p1": "node dist/naisys.js ./agents/webdev-battle/player1.yaml",
-    "run agent:p2": "node dist/naisys.js ./agents/webdev-battle/player2.yaml",
+    "agent:dev": "node dist/naisys.js ./agents/3-team-dev-db-content/dev.yaml",
+    "agent:db": "node dist/naisys.js ./agents/3-team-dev-db-content/db.yaml",
+    "agent:content": "node dist/naisys.js ./agents/3-team-dev-db-content/content.yaml",
     "clean": "rm -rf dist",
-    "compile": "tsc --build --verbose",
+    "compile": "tsc",
     "eslint": "npx eslint --rulesdir eslint-rules src",
     "test": "tsc && node --experimental-vm-modules node_modules/jest/bin/jest.js --testPathPattern=dist/__tests__",
     "prettier": "npx prettier --write .",
     "dependency-graph": "madge --image dependency-graph.png dist",
     "detect-cycles": "madge --circular dist",
     "updates:check": "npm-check-updates",
-    "updates:apply": "npm-check-updates -u && npm update",
-    "npm:publish:dryrun": "npm run clean && npm run compile && npm publish --dry-run",
+    "updates:apply": "npm-check-updates -u && npm install",
+    "npm:publish:dryrun": "npm run clean && npm install && npm run compile && npm publish --dry-run",
     "postinstall": "chmod +x ./bin/*"
   },
   "repository": {
@@ -42,7 +43,7 @@
   "devDependencies": {
     "@types/escape-html": "1.0.4",
     "@types/js-yaml": "4.0.9",
-    "@types/node": "20.11.26",
+    "@types/node": "20.11.28",
     "@types/text-table": "0.2.5",
     "@typescript-eslint/eslint-plugin": "7.2.0",
     "@typescript-eslint/parser": "7.2.0",
@@ -53,14 +54,14 @@
     "typescript": "5.4.2"
   },
   "dependencies": {
-    "@anthropic-ai/sdk": "0.17.2",
+    "@anthropic-ai/sdk": "0.18.0",
     "@google/generative-ai": "0.3.0",
     "chalk": "5.3.0",
     "commander": "12.0.0",
     "dotenv": "16.4.5",
     "escape-html": "1.0.3",
     "js-yaml": "4.1.0",
-    "openai": "4.28.4",
+    "openai": "4.29.0",
     "sqlite": "5.1.1",
     "sqlite3": "5.1.7",
     "text-table": "0.2.0",