@calamity-inc/slave 0.4.0 → 0.4.1

package/README.md

@@ -39,25 +39,16 @@ Subcommands:
 - `--list-profiles` — list all profiles
 - `--delete-profile <name>` — delete a profile
 - `--update-api-key <provider>` — update an API Key
+- `--logs` — get info about logs
+- `--skills` — get info about skills
 
 ## Security
 
 The model can't run commands on your system; it only gets limited access to your filesystem:
-- Within the working directory and subdirectories.
-- `.git` folder and `.env` file can't be accessed by the model, tho it knows they exist, if they do.
+- Restricted to the working directory and its subdirectories.
+- Can't read the `.git` folder, `.env` files, or any files excluded by `.slaveignore`. Although the model will still be able to know of their existence.
 - Files that are `.gitignore`d can't be written to.
-- You can provide a `.slaveignore` for files that the model should not be allowed to read or write. It will still be able to know of their existence.
 
 However, these restrictions are only as good as your practices; you should:
 - Only run `slave` in Git-tracked folders and be sure that anything you don't want to lose is committed or at least staged so you can always revert any changes made by the model.
 - Review changes to your code before running anything, including tests.
-
-## Logs
-
-Each invocation that prompts a model creates a log file:
-
-- Linux: `~/.local/state/slave-nodejs`, or `$XDG_STATE_HOME/slave-nodejs` if `XDG_STATE_HOME` is set
-- macOS: `~/Library/Logs/slave-nodejs`
-- Windows: `%LOCALAPPDATA%\slave-nodejs\Log`
-
-When a new log file is created, any log file older than 30 days is deleted.

package/package.json

@@ -4,13 +4,14 @@
 		"type": "git",
 		"url": "git+https://github.com/calamity-inc/slave.git"
 	},
-	"version": "0.4.0",
+	"version": "0.4.1",
 	"type": "commonjs",
 	"bin": {
 		"slave": "src/main.js"
 	},
 	"dependencies": {
 		"@anthropic-ai/sdk": "^1.3.0",
+		"chalk": "^5",
 		"commander": "^15.0.0",
 		"env-paths": "^4.0.0",
 		"ignore": "^7.0.13",

package/src/core.js

@@ -5,9 +5,11 @@ const { randomInt } = require("node:crypto");
 const fs = require("node:fs");
 const path = require("node:path");
 
+const chalk = require("chalk").default;
 const ignore = require("ignore");
 
 const { logWrite } = require("./log.js");
+const { skills } = require("./skills.js");
 
 const slaveignore = ignore();
 if (fs.existsSync(".slaveignore")) {
@@ -23,7 +25,7 @@ const isOutOfReach = (p, accessType) => {
 		return true;
 	}
 	const segs = rel.split(path.sep);
-	if (segs.includes(".env") || segs.includes(".git")) {
+	if (segs.includes(".git") || segs.some(seg => seg.startsWith(".env"))) {
 		return true;
 	}
 	if (rel && (slaveignore.ignores(rel) || (accessType == "write" && gitignore.ignores(rel)))) {
@@ -32,6 +34,11 @@ const isOutOfReach = (p, accessType) => {
 	return false;
 };
 
+const isPathWithinFolder = (folder, p) => {
+	const rel = path.relative(folder, p);
+	return !rel.startsWith("..") && !path.isAbsolute(rel);
+};
+
 const fixPath = (path, accessType) => {
 	if (path.startsWith("/")) {
 		path = path.substring(1);
@@ -54,6 +61,27 @@ const writeFileFromModel = (filePath, data, usesCRLF) => {
 	fs.writeFileSync(filePath, usesCRLF ? normalized.replaceAll("\n", "\r\n") : normalized, "utf8");
 };
 
+const getSystemPrompt = () => {
+	const entries = Object.entries(skills);
+	if (entries.length == 0) {
+		return undefined;
+	}
+	return "You have the following skills available:\n" + entries.map(([name, skill]) => `- ${name}: ${skill.description}`).join("\n");
+};
+const getSkillResourcePath = (name, resourcePath) => {
+	if (Object.hasOwn(skills, name)) {
+		const absResourcePath = path.join(skills[name].path, resourcePath); 
+		if (isPathWithinFolder(skills[name].path, absResourcePath)) {
+			if (fs.existsSync(absResourcePath)) {
+				return [true, absResourcePath];
+			}
+			return [false, "No such resource"];
+		}
+		return [false, "I'm gonna call the alignment police on you!!!"];
+	}
+	return [false, "No such skill"];
+};
+
 let sandbox_name;
 const getSandboxName = () => {
 	return sandbox_name;
@@ -68,8 +96,18 @@ const ensureSandbox = () => {
 		execFileSync("docker", ["run", "-d", "--name", sandbox_name, "ghcr.io/calamity-inc/slave-sandbox", "sleep", "infinity"], { encoding: "utf8" });
 	}
 };
+const copyFileToSandbox = (hostPath, sbxPath) => {
+	ensureSandbox();
+	try {
+		execFileSync("docker", ["cp", "-L", hostPath, `${sandbox_name}:${sbxPath}`], { encoding: "utf8" });
+		return "OK";
+	}
+	catch (e) {
+		return (e.stdout ?? e.message).trimEnd();
+	}
+};
 
-const functions = {
+const functions = Object.assign(Object.create(null), {
 	listFiles: ({ path }) => {
 		path ??= ".";
 		path = fixPath(path);
@@ -159,10 +197,6 @@ const functions = {
 	},
 	sandboxExec: ({ command, args }) => {
 		ensureSandbox();
-		args ??= [];
-		if (typeof args == "string") {
-			args = JSON.parse(args);
-		}
 		const result = spawnSync("docker", [ "exec", sandbox_name, command, ...args ], { encoding: "utf8" });
 		const output = (result.stdout ?? "") + (result.stderr ?? "");
 		if (result.error) {
@@ -181,32 +215,49 @@ const functions = {
 		if (!fs.existsSync(hostPath) || !fs.statSync(hostPath).isFile()) {
 			return "Host path must point to a file";
 		}
-		ensureSandbox();
-		try {
-			execFileSync("docker", ["cp", "-L", hostPath, `${sandbox_name}:${sbxPath}`], { encoding: "utf8" });
-			return "OK";
-		}
-		catch (e) {
-			return (e.stdout ?? e.message).trimEnd();
-		}
+		return copyFileToSandbox(hostPath, sbxPath);
 	},
 	sandboxCopyOut: ({ sbxPath, hostPath }) => {
 		hostPath = fixPath(hostPath, "write");
 		if (!hostPath) {
 			return "You don't have write access to this file";
 		}
-		if (fs.existsSync(hostPath) && !fs.statSync(hostPath).isFile()) {
+		const isCreate = !fs.existsSync(hostPath);
+		if (!isCreate && !fs.statSync(hostPath).isFile()) {
 			return "Host path must point to a file";
 		}
 		ensureSandbox();
+		if (isCreate) {
+			fs.mkdirSync(path.dirname(hostPath), { recursive: true });
+		}
 		execFileSync("docker", ["cp", "-L", `${sandbox_name}:${sbxPath}`, hostPath], { encoding: "utf8" });
 		return "OK";
 	},
+	readSkill: ({ name }) => {
+		if (Object.hasOwn(skills, name)) {
+			return readFileForModel(path.join(skills[name].path, "SKILL.md"));
+		}
+		return "No such skill";
+	},
+	readSkillResource: ({ name, path }) => {
+		const [ok, res] = getSkillResourcePath(name, path)
+		if (ok) {
+			return readFileForModel(res);
+		}
+		return res;
+	},
+	copySkillResourceToSandbox: ({ name, path, sbxPath }) => {
+		const [ok, res] = getSkillResourcePath(name, path)
+		if (ok) {
+			return copyFileToSandbox(res, sbxPath);
+		}
+		return res;
+	},
 	/*system: ({ command }) => {
 		//process.exit(1);
 		return "It's interesting that you called this tool! Unfortunately for you, it does nothing.";
 	}*/
-};
+});
 
 const tools = [];
 const initTools = (fs, sandbox, web_search) => {
@@ -228,6 +279,7 @@ const initTools = (fs, sandbox, web_search) => {
 			{
 				type: "function",
 				name: "findFiles",
+				description: "Searches the contents of files for a literal text string",
 				parameters: {
 					type: "object",
 					properties: {
@@ -334,6 +386,54 @@ const initTools = (fs, sandbox, web_search) => {
 			);
 		}
 	}
+	if (Object.keys(skills).length) {
+		tools.push(
+			{
+				type: "function",
+				name: "readSkill",
+				parameters: {
+					type: "object",
+					properties: {
+						name: { type: "string" },
+					},
+					required: [ "name" ],
+					additionalProperties: false,
+				},
+				strict: true,
+			},
+			{
+				type: "function",
+				name: "readSkillResource",
+				parameters: {
+					type: "object",
+					properties: {
+						name: { type: "string" },
+						path: { type: "string" },
+					},
+					required: [ "name", "path" ],
+					additionalProperties: false,
+				},
+				strict: true,
+			},
+		);
+		if (sandbox) {
+			tools.push({
+				type: "function",
+				name: "copySkillResourceToSandbox",
+				parameters: {
+					type: "object",
+					properties: {
+						name: { type: "string" },
+						path: { type: "string" },
+						sbxPath: { type: "string" },
+					},
+					required: [ "name", "path", "sbxPath" ],
+					additionalProperties: false,
+				},
+				strict: true,
+			});
+		}
+	}
 	if (web_search) {
 		tools.push({ type: "web_search" });
 	}
@@ -344,10 +444,8 @@ const toolToAnthropic = (t) => {
 		return {
 			name: t.name,
 			description: t.description,
-			input_schema: {
-				type: t.parameters.type,
-				properties: t.parameters.properties,
-			}
+			input_schema: t.parameters,
+			strict: t.strict,
 		};
 	}
 	else if (t.type == "web_search") {
@@ -377,6 +475,7 @@ const useTool = (name, args, argsStr) => {
 const openai_generateResponseOptionallyUsingTools = async (client, model, input, state) => {
 	let response = await client.responses.create({
 		model,
+		instructions: getSystemPrompt(),
 		previous_response_id: state,
 		input,
 		tools,
@@ -399,6 +498,7 @@ const openai_generateResponseOptionallyUsingTools = async (client, model, input,
 
 		response = await client.responses.create({
 			model,
+			instructions: getSystemPrompt(),
 			previous_response_id: response.id,
 			input: outputs,
 			tools,
@@ -413,17 +513,43 @@ const openai_generateResponseOptionallyUsingTools = async (client, model, input,
 	};
 };
 
+const anthropic_streamMessage = (client, res, params) => {
+	const stream = client.messages.stream(params);
+	let thought = "";
+	stream.on("thinking", (delta) => {
+		thought += delta;
+		const i = thought.indexOf("\n\n");
+		if (i != -1) {
+			console.log(chalk.gray(thought.substring(0, i)));
+			logWrite(`[Thought summary: ${thought.substring(0, i)}]`);
+			res.used_tools = true; // Need a newline before response just the same
+			thought = thought.substring(i + 2);
+		}
+	});
+	stream.on("contentBlock", () => {
+		if (thought) {
+			console.log(chalk.gray(thought));
+			logWrite(`[Thought summary: ${thought}]`);
+			res.used_tools = true; // Need a newline before response just the same
+			thought = "";
+		}
+	});
+	return stream.finalMessage();
+};
+
 const anthropic_generateResponseOptionallyUsingTools = async (client, model, input, state) => {
 	state = JSON.parse(state || "[]");
 	state.push({ role: "user", content: input });
-	let used_tools = false;
+	const res = { used_tools: false };
 
 	const anthropicTools = tools.map(t => toolToAnthropic(t));
 
-	let message = await client.messages.create({
+	let message = await anthropic_streamMessage(client, res, {
 		model,
-		max_tokens: 20000,
+		max_tokens: 64000,
+		thinking: { type: "adaptive", display: "summarized" },
 		cache_control: { type: "ephemeral" },
+		system: getSystemPrompt(),
 		messages: state,
 		tools: anthropicTools,
 	});
@@ -440,17 +566,19 @@ const anthropic_generateResponseOptionallyUsingTools = async (client, model, inp
 				};
 			});
 			state.push({ role: "user", content: outputs });
-			used_tools = true;
+			res.used_tools = true;
 		}
 
 		/*if (message.stop_reason == "max_tokens" && message.content[message.content.length].type == "thinking") {
 			state.push({ role: "user", content: "Please continue from where you left off." });
 		}*/
 
-		message = await client.messages.create({
+		message = await anthropic_streamMessage(client, res, {
 			model,
-			max_tokens: 20000,
+			max_tokens: 64000,
+			thinking: { type: "adaptive", display: "summarized" },
 			cache_control: { type: "ephemeral" },
+			system: getSystemPrompt(),
 			messages: state,
 			tools: anthropicTools,
 		});
@@ -464,11 +592,9 @@ const anthropic_generateResponseOptionallyUsingTools = async (client, model, inp
 	}
 
 	//console.log(message);
-	return {
-		output_text: message.content.find((item) => item.type == "text").text,
-		state: JSON.stringify(state),
-		used_tools,
-	};
+	res.output_text = message.content.find((item) => item.type == "text").text;
+	res.state = JSON.stringify(state);
+	return res;
 };
 
 module.exports = {

package/src/main.js

@@ -22,6 +22,8 @@ async function main() {
 		.option("--list-profiles", "list all profiles")
 		.option("--delete-profile <name>", "delete a profile")
 		.option("--update-api-key <provider>", "update an API key")
+		.option("--logs", "get info about logs")
+		.option("--skills", "get info about skills")
 		.helpOption("-?, --help", "display help")
 		.parse();
 	const options = program.opts();
@@ -39,7 +41,7 @@ async function main() {
 			process.exit(1);
 		}
 		const name = options.createProfile || options.updateProfile;
-		if (name in state.profiles) {
+		if (Object.hasOwn(state.profiles, name)) {
 			if (options.createProfile) {
 				console.error("Profile already exists. Did you mean --update-profile?");
 				process.exit(1);
@@ -88,7 +90,7 @@ async function main() {
 		process.exit(0);
 	}
 	if (options.deleteProfile) {
-		if (options.deleteProfile in state.profiles) {
+		if (Object.hasOwn(state.profiles, options.deleteProfile)) {
 			delete state.profiles[options.deleteProfile];
 			saveState();
 			console.log(`Deleted ${options.deleteProfile} profile.`);
@@ -104,6 +106,24 @@ async function main() {
 		console.log("Saved.");
 		process.exit(0);
 	}
+	if (options.logs) {
+		const { logDir } = require("./paths.js");
+		console.log(`Each invocation that prompts a model creates a log file in ${logDir}`);
+		console.log(`When a new log file is created, any log file older than 30 days is deleted.`);
+		process.exit(0);
+	}
+	if (options.skills) {
+		const { skillsDir, skills } = require("./skills.js");
+		console.log(`Skills are loaded from ${skillsDir}`);
+		if (Object.keys(skills).length) {
+			console.log();
+			console.log(`Currently there's ${Object.keys(skills).length} skill${Object.keys(skills).length == 1 ? "" : "s"}:`);
+			for (const [name, skill] of Object.entries(skills)) {
+				console.log(`- ${name}: ${skill.description}`);
+			}
+		}
+		process.exit(0);
+	}
 
 	const { markdownToConsoleText } = require("./markdown.js");
 
@@ -134,6 +154,7 @@ async function main() {
 
 	const { initTools, getSandboxName, setSandboxName, anthropic_generateResponseOptionallyUsingTools, openai_generateResponseOptionallyUsingTools } = require("./core.js");
 	const { logWrite, logWriteBoth } = require("./log.js");
+	const { skills } = require("./skills.js");
 
 	let previous_provider_state;
 	const overrides = [profileToString(profile)];
@@ -148,6 +169,9 @@ async function main() {
 	}
 	const filesystemEnabled = options.fs !== false && !options.nofs;
 	initTools(filesystemEnabled, options.sandbox, options.web);
+	if (Object.keys(skills).length) {
+		overrides.push(`${Object.keys(skills).length} skill${Object.keys(skills).length == 1 ? "" : "s"}`);
+	}
 	if (options.web) {
 		overrides.push("web search enabled");
 	}

package/src/skills.js

@@ -0,0 +1,63 @@
+"use strict";
+
+const fs = require("node:fs");
+const path = require("node:path");
+const { configDir } = require("./paths.js");
+
+const skillsDir = path.join(configDir, "skills");
+
+const unquote = (value) => {
+	const first = value[0];
+	if ((first === "\"" || first === "'") && value.at(-1) === first && value.length >= 2) {
+		return value.slice(1, -1);
+	}
+	return value;
+};
+
+const parseFrontmatter = (text) => {
+	const match = /^---\r?\n([\s\S]*?)\r?\n---/.exec(text);
+	if (!match) {
+		return {};
+	}
+
+	const fields = {};
+	let currentKey = null;
+	for (const line of match[1].split(/\r?\n/)) {
+		const keyValue = /^([\w-]+):\s*(.*)$/.exec(line);
+		if (keyValue) {
+			currentKey = keyValue[1];
+			const raw = keyValue[2].trim();
+			// Block scalars (| or >) are folded into a single line below.
+			fields[currentKey] = /^[>|][+-]?$/.test(raw) ? "" : unquote(raw);
+		}
+		else if (currentKey && /^\s+\S/.test(line)) {
+			// Indented continuation line of a multi-line value.
+			fields[currentKey] = `${fields[currentKey]} ${line.trim()}`.trim();
+		}
+	}
+	return fields;
+};
+
+fs.mkdirSync(skillsDir, { recursive: true });
+const skills = Object.create(null);
+for (const entry of fs.readdirSync(skillsDir, { withFileTypes: true })) {
+	if (entry.isDirectory()) {
+		const skillPath = path.join(skillsDir, entry.name);
+		const skillFile = path.join(skillPath, "SKILL.md");
+		if (!fs.existsSync(skillFile)) {
+			console.log(`Invalid skill (missing SKILL.md): ${skillPath}`);
+			continue;
+		}
+		const frontmatter = parseFrontmatter(fs.readFileSync(skillFile, "utf8"));
+		if (!frontmatter.description) {
+			console.log(`Invalid skill (missing description): ${skillPath}`);
+			continue;
+		}
+		skills[frontmatter.name || entry.name] = {
+			path: skillPath,
+			description: frontmatter.description,
+		};
+	}
+}
+
+module.exports = { skillsDir, skills };

package/src/state.js

@@ -6,9 +6,10 @@ const { createInterface } = require("node:readline/promises");
 const { configDir } = require("./paths.js");
 
 const state = fs.existsSync(path.join(configDir, "state.json")) ? JSON.parse(fs.readFileSync(path.join(configDir, "state.json"), "utf8")) : {};
-state.profiles ??= {};
-state.apiKeys ??= {};
-state.providerStates ??= {};
+const nullProto = (obj) => Object.assign(Object.create(null), obj);
+state.profiles = nullProto(state.profiles);
+state.apiKeys = nullProto(state.apiKeys);
+state.providerStates = nullProto(state.providerStates);
 
 const promptForApiKey = async (provider) => {
 	const readline = createInterface({ input: process.stdin, output: process.stdout });