@tyvm/knowhow 0.0.129 → 0.0.131

package/src/index.ts

@@ -1,14 +1,8 @@
 import { summarizeFiles, summarizeFile } from "./ai";
-import {
-  saveAllFileHashes,
-  saveHashes,
-  getHashes,
-  checkNoFilesChanged,
-} from "./hashes";
+import type { AgentOptions } from "./ai";
 import * as fs from "fs";
 import * as path from "path";
 import * as crypto from "crypto";
-import { promisify } from "util";
 import { globSync } from "glob";
 
 import { Prompts } from "./prompts";
@@ -37,12 +31,25 @@ import {
   getConfiguredEmbeddingMap,
 } from "./embeddings";
 
-import { abort } from "process";
 import { convertToText } from "./conversion";
 import { knowhowMcpClient } from "./services/Mcp";
 import { services } from "./services/";
 import { Models } from "./types";
 
+// ---------------------------------------------------------------------------
+// Re-export generate pipeline (moved to src/generate.ts)
+// ---------------------------------------------------------------------------
+export {
+  generate,
+  GenerateOptions,
+  buildWaves,
+  normalizeInputPattern,
+  withOutputTarget,
+  writeAgentOrSummaryOutput,
+  handleMultiOutputGeneration,
+  handleSingleOutputGeneration,
+} from "./generate";
+
 export * as clients from "./clients";
 export * as agents from "./agents";
 export * as services from "./services";
@@ -60,11 +67,16 @@ export { ConversionService } from "./services/conversion/ConversionService";
 // Export plugin types for external plugins
 export { PluginBase } from "./plugins/PluginBase";
 export { PluginMeta, Plugin, PluginContext } from "./plugins/types";
+export { PluginService } from "./plugins/plugins";
+export { SkillsPlugin } from "./plugins/SkillsPlugin";
 // Export embedding types
 export { MinimalEmbedding, Embeddable } from "./types";
 
+// ---------------------------------------------------------------------------
+// Embed
+// ---------------------------------------------------------------------------
+
 export async function embed() {
-  // load config
   const config = await getConfig();
   const ignorePattern = await getIgnorePattern();
 
@@ -72,7 +84,6 @@ export async function embed() {
     config.embeddingModel || EmbeddingModels.openai.EmbeddingAda2;
 
   if (!config.embedSources) {
-    // No embeddings configured
     return;
   }
 
@@ -81,6 +92,10 @@ export async function embed() {
   }
 }
 
+// ---------------------------------------------------------------------------
+// Purge
+// ---------------------------------------------------------------------------
+
 export async function purge(globPath: string) {
   const files = globSync(globPath);
   const embeddings = await getConfiguredEmbeddingMap();
@@ -107,6 +122,10 @@ export async function purge(globPath: string) {
   }
 }
 
+// ---------------------------------------------------------------------------
+// Upload
+// ---------------------------------------------------------------------------
+
 export async function upload() {
   const config = await getConfig();
   const { AwsS3, Embeddings, knowhowApiClient } = services();
@@ -124,10 +143,6 @@ export async function upload() {
     }
     const items = JSON.parse(await readFile(source.output, "utf8"));
     const { name: embeddingName } = path.parse(source.output);
-    const data = {
-      embeddingName,
-      items,
-    };
 
     if (Embeddings.hasResolver(source.remoteType) && source.remoteType !== "knowhow") {
       console.log(
@@ -142,7 +157,6 @@ export async function upload() {
       if (!source.remoteId) {
         throw new Error("remoteId is required for knowhow uploads");
       }
-      // Warn if the local embeddingModel differs from the one stored on the backend
       try {
         const remoteEmbedding = await knowhowApiClient.getOrgEmbedding(source.remoteId);
         const localModel = config.embeddingModel || EmbeddingModels.openai.EmbeddingAda2;
@@ -157,12 +171,11 @@ export async function upload() {
           );
         }
       } catch (e) {
-        // Non-fatal — don't block upload if metadata fetch fails
+        // Non-fatal
       }
       const url = await knowhowApiClient.getPresignedUploadUrl(source);
       console.log("Uploading to", url);
       await AwsS3.uploadToPresignedUrl(url, source.output);
-      // Sync config metadata back to the backend DB
       await knowhowApiClient.updateEmbeddingMetadata(source.remoteId, {
         inputGlob: source.input,
         outputPath: source.output,
@@ -181,201 +194,9 @@ export async function upload() {
   }
 }
 
-/**
- * Normalizes an input pattern to a valid glob pattern.
- * Supports:
- *   - Standard glob patterns (e.g. "src/**\/*.ts")
- *   - Brace expansion (e.g. "{src/a.ts,src/b.ts}")
- *   - Comma-separated file paths (e.g. "src/a.ts,src/b.ts") — auto-converted to brace expansion
- *   - Mixed comma-separated list with globs (e.g. "src/a.ts,src/commands/**\/*.ts")
- */
-function normalizeInputPattern(input: string): string {
-  // If it already has braces, use as-is (already brace-expanded)
-  if (input.includes("{")) {
-    return input;
-  }
-  // If it contains commas, treat as comma-separated list and wrap in braces
-  // This also handles the mixed case: "src/a.ts,src/commands/**/*.ts"
-  if (input.includes(",")) {
-    const parts = input.split(",").map((p) => p.trim());
-    return `{${parts.join(",")}}`;
-  }
-  return input;
-}
-
-export async function generate(): Promise<void> {
-  const config = await getConfig();
-  for (const source of config.sources) {
-    console.log("Generating", source.input, "to", source.output);
-    if (source.kind === "file" || !source.kind) {
-      const files = globSync(normalizeInputPattern(source.input));
-      const prompt = await loadPrompt(source.prompt);
-
-      if (source.output.endsWith("/")) {
-        await handleMultiOutputGeneration(
-          source.model,
-          source.input,
-          files,
-          prompt,
-          source.output,
-          source.outputExt,
-          source.outputName,
-          source.kind,
-          source.agent
-        );
-      } else {
-        await handleSingleOutputGeneration(
-          source.model,
-          files,
-          prompt,
-          source.output,
-          source.kind,
-          source.agent
-        );
-      }
-    } else {
-      await handleAllKindsGeneration(source);
-    }
-  }
-}
-
-async function handleAllKindsGeneration(source: GenerationSource) {
-  const { Plugins } = services();
-  const { kind, input } = source;
-  if (Plugins.isPlugin(kind)) {
-    const data = await Plugins.call(kind, input);
-    if (source.output.endsWith("/")) {
-      throw new Error(`Plugin ${kind} can only output to a single file`);
-    }
-    await writeFile(source.output, data);
-  }
-  return handleFileKindGeneration(source);
-}
-
-async function handleFileKindGeneration(source: GenerationSource) {
-  const prompt = await loadPrompt(source.prompt);
-  const files = globSync(normalizeInputPattern(source.input));
-  console.log("Analyzing files: ", files);
-
-  if (source.output.endsWith("/")) {
-    await handleMultiOutputGeneration(
-      source.model,
-      source.input,
-      files,
-      prompt,
-      source.output,
-      source.outputExt,
-      source.outputName,
-      source.kind,
-      source.agent
-    );
-  } else {
-    await handleSingleOutputGeneration(
-      source.model,
-      files,
-      prompt,
-      source.output,
-      source.kind,
-      source.agent
-    );
-  }
-}
-export async function handleMultiOutputGeneration(
-  model: string,
-  inputPattern: string,
-  files: string[],
-  prompt: string,
-  output: string,
-  outputExt = "mdx",
-  outputName?: string,
-  kind?: string,
-  agent?: string
-) {
-  // get the hash of the prompt
-  const promptHash = crypto.createHash("md5").update(prompt).digest("hex");
-
-  // get the files matching the input pattern
-  const hashes = await getHashes();
-
-  const inputPath = inputPattern.includes("**")
-    ? inputPattern.split("**")[0]
-    : "";
-
-  for (const file of files) {
-    // get the hash of the file
-    const fileContent = await convertToText(file);
-    const fileHash = crypto.createHash("md5").update(fileContent).digest("hex");
-
-    if (!hashes[file]) {
-      hashes[file] = { promptHash: "", fileHash: "" };
-    }
-
-    // write the summary to the output file
-    const { name, ext, dir } = path.parse(file);
-    const nestedFolder = inputPath ? (dir + "/").replace(inputPath, "") : "";
-    const outputFolder = path.join(output, nestedFolder);
-
-    if (!fs.existsSync(outputFolder)) {
-      fs.mkdirSync(outputFolder, { recursive: true });
-    }
-
-    const outputFileName = outputName || name;
-    const outputFile = path.join(
-      outputFolder,
-      outputFileName + "." + outputExt
-    );
-    console.log({ dir, inputPath, nestedFolder, outputFile });
-
-    const toCheck = [file, outputFile];
-    const noChanges = await checkNoFilesChanged(toCheck, promptHash, hashes);
-    if (noChanges) {
-      console.log("Skipping file", file, "because it hasn't changed");
-      continue;
-    }
-
-    // summarize the file
-    console.log("Summarizing", file);
-    const summary = prompt
-      ? await summarizeFile(file, prompt, model, agent)
-      : fileContent;
-
-    console.log("Writing summary to", outputFile);
-    await writeFile(outputFile, summary);
-
-    await saveAllFileHashes(toCheck, promptHash);
-  }
-}
-
-export async function handleSingleOutputGeneration(
-  model: string,
-  files: string[],
-  prompt: string,
-  outputFile: string,
-  kind?: string,
-  agent?: string
-) {
-  const hashes = await getHashes();
-  const promptHash = crypto.createHash("md5").update(prompt).digest("hex");
-
-  const filesToCheck = [outputFile, ...files];
-  const noChanges = await checkNoFilesChanged(filesToCheck, promptHash, hashes);
-  if (noChanges) {
-    console.log(`Skipping ${files.length} files because they haven't changed`);
-    return;
-  }
-
-  console.log("Summarizing", files.length, "files");
-  const summary = prompt
-    ? await summarizeFiles(files, prompt, model, agent)
-    : (await Promise.all(files.map(convertToText))).join("\n\n");
-
-  const fileHash = crypto.createHash("md5").update(summary).digest("hex");
-
-  console.log("Writing summary to", outputFile);
-  await writeFile(outputFile, summary);
-
-  await saveAllFileHashes(filesToCheck, promptHash);
-}
+// ---------------------------------------------------------------------------
+// Download
+// ---------------------------------------------------------------------------
 
 export async function download() {
   const config = await getConfig();
@@ -419,10 +240,7 @@ export async function download() {
         "to",
         destinationPath
       );
-      const preSignedUrl = await knowhowApiClient.getPresignedDownloadUrl(
-        source
-      );
-      // Ensure output directory exists
+      const preSignedUrl = await knowhowApiClient.getPresignedDownloadUrl(source);
       const outputDir = path.dirname(destinationPath);
       if (!fs.existsSync(outputDir)) fs.mkdirSync(outputDir, { recursive: true });
       await AwsS3.downloadFromPresignedUrl(preSignedUrl, destinationPath);

package/src/plugins/PluginBase.ts

@@ -1,4 +1,3 @@
-import { MinimalEmbedding } from "../types";
 import { Plugin, PluginContext, PluginMeta } from "./types";
 
 export abstract class PluginBase implements Plugin {
@@ -66,7 +65,6 @@ export abstract class PluginBase implements Plugin {
   /** Mandatory plugin actions ---------------------------------------- */
   /* ------------------------------------------------------------------ */
   abstract call(input?: string): Promise<string>;
-  abstract embed(input: string): Promise<MinimalEmbedding[]>;
 }
 
 export { PluginMeta, Plugin } from "./types";

package/src/plugins/SkillsPlugin.ts

@@ -1,33 +1,11 @@
-import { promises as fs, Dirent } from "fs";
-import * as path from "path";
 import { PluginBase, PluginMeta } from "./PluginBase";
 import { Plugin, PluginContext } from "./types";
-import { MinimalEmbedding } from "../types";
-import { getConfig } from "../config";
-
-interface SkillMeta {
-  name: string;
-  description: string;
-  filePath: string;
-}
-
-function parseFrontmatter(content: string): Record<string, string> {
-  const match = content.match(/^---\n([\s\S]*?)\n---/);
-  if (!match) return {};
-  const result: Record<string, string> = {};
-  for (const line of match[1].split("\n")) {
-    const [key, ...rest] = line.split(":");
-    if (key && rest.length) result[key.trim()] = rest.join(":").trim();
-  }
-  return result;
-}
 
 export class SkillsPlugin extends PluginBase implements Plugin {
   static readonly meta: PluginMeta = {
     key: "skills",
     name: "Skills Plugin",
-    description:
-      "Scans configured skills directories for SKILL.md files and provides skill content to agents",
+    description: "Lists available skills",
     requires: [],
   };
 
@@ -37,114 +15,23 @@ export class SkillsPlugin extends PluginBase implements Plugin {
     super(context);
   }
 
-  async call(input?: string): Promise<string> {
-    const result = await this.embed(input || "");
-    return result.map((e) => e.text).join("\n\n");
-  }
-
-  async embed(userPrompt: string): Promise<MinimalEmbedding[]> {
-    const config = await getConfig();
-    const skillDirs: string[] = (config as any).skills || [];
-
-    if (skillDirs.length === 0) {
-      return [];
-    }
-
-    const skills = await this.scanSkills(skillDirs);
+  /**
+   * Returns a summary list of in-memory skills (name + description).
+   * This is called by the plugin system to inject context into agent prompts.
+   * Skills must be loaded into memory first via Behaviors.initFromDisk() at startup.
+   */
+  async call(_input?: string): Promise<string> {
+    const { Behaviors } = this.context;
+    const skills = Behaviors.listSkills();
 
     if (skills.length === 0) {
-      return [];
-    }
-
-    // Check if any skill name appears in the prompt
-    const matchedSkills = skills.filter((skill) =>
-      userPrompt.toLowerCase().includes(skill.name.toLowerCase())
-    );
-
-    if (matchedSkills.length > 0) {
-      // Return full skill content for matched skills
-      const results: MinimalEmbedding[] = [];
-      for (const skill of matchedSkills) {
-        try {
-          const content = await fs.readFile(skill.filePath, "utf-8");
-          results.push({
-            id: skill.filePath,
-            text: `## Skill: ${skill.name}\nFile: ${skill.filePath}\n\n${content}`,
-            metadata: { filePath: skill.filePath },
-          });
-        } catch {
-          // Skip unreadable files
-        }
-      }
-      return results;
-    }
-
-    // Return skill discovery summary
-    const summary = this.buildSkillSummary(skills);
-    return [
-      {
-        id: "skills:summary",
-        text: summary,
-        metadata: {},
-      },
-    ];
-  }
-
-  private async scanSkills(dirs: string[]): Promise<SkillMeta[]> {
-    const skills: SkillMeta[] = [];
-    for (const dir of dirs) {
-      const resolvedDir = dir.replace(/^~/, process.env.HOME || "");
-      try {
-        const found = await this.findSkillFiles(resolvedDir);
-        skills.push(...found);
-      } catch {
-        // Skip directories that don't exist or can't be read
-      }
-    }
-    return skills;
-  }
-
-  private async findSkillFiles(dir: string): Promise<SkillMeta[]> {
-    const skills: SkillMeta[] = [];
-    let entries: Dirent[];
-    try {
-      entries = await fs.readdir(dir, { withFileTypes: true });
-    } catch {
-      return skills;
-    }
-
-    for (const entry of entries) {
-      const fullPath = path.join(dir, entry.name);
-      if (entry.isDirectory()) {
-        const nested = await this.findSkillFiles(fullPath);
-        skills.push(...nested);
-      } else if (entry.isFile() && entry.name === "SKILL.md") {
-        try {
-          const content = await fs.readFile(fullPath, "utf-8");
-          const frontmatter = parseFrontmatter(content);
-          const name = frontmatter.name;
-          const description = frontmatter.description || "";
-          if (name) {
-            skills.push({ name, description, filePath: fullPath });
-          }
-        } catch {
-          // Skip unreadable files
-        }
-      }
+      return "No skills available";
     }
 
-    return skills;
-  }
-
-  private buildSkillSummary(skills: SkillMeta[]): string {
     const lines = ["Available skills:"];
     for (const skill of skills) {
-      lines.push(`- ${skill.name} (${skill.filePath}): ${skill.description}`);
+      lines.push(`- ${skill.name}: ${skill.description}`);
     }
-    lines.push("");
-    lines.push(
-      "To use a skill, reference its name in your request and I will load the full instructions."
-    );
     return lines.join("\n");
   }
 }

package/src/plugins/language.ts

@@ -6,7 +6,6 @@ import { getConfig, getLanguageConfig } from "../config";
 import { getEnabledPlugins } from "../types";
 import { PluginBase, PluginMeta } from "./PluginBase";
 import { Plugin, PluginContext } from "./types";
-import { PluginService } from "./plugins";
 
 /**
  * Language Plugin with event-driven context loading

package/src/plugins/plugins.ts

@@ -14,24 +14,12 @@ import { SkillsPlugin } from "./SkillsPlugin";
 import { EventService } from "../services/EventService";
 
 export class PluginService {
-  private pluginMap = new Map<string, Plugin>();
+  protected pluginMap = new Map<string, Plugin>();
   private events?: EventService;
 
   constructor(context: PluginContext) {
     this.events = context.Events;
     context.Plugins = this;
-
-    // Register migrated PluginBase plugins
-    this.pluginMap.set("embeddings", new EmbeddingPlugin(context));
-    this.pluginMap.set("vim", new VimPlugin(context));
-    this.pluginMap.set("linter", new LinterPlugin(context));
-    this.pluginMap.set("language", new LanguagePlugin(context));
-    this.pluginMap.set("url", new UrlPlugin(context));
-    this.pluginMap.set("git", new GitPlugin(context));
-    this.pluginMap.set("tmux", new TmuxPlugin(context));
-    this.pluginMap.set("agents-md", new AgentsMdPlugin(context));
-    this.pluginMap.set("exec", new ExecPlugin(context));
-    this.pluginMap.set("skills", new SkillsPlugin(context));
   }
 
   /* -------- lifecycle helpers ------------------------------------ */
@@ -146,3 +134,26 @@ export class PluginService {
     return newPlugin.embed ? newPlugin.embed(userInput) : [];
   }
 }
+
+/**
+ * CliPluginService extends PluginService with the full set of default plugins
+ * used by the Knowhow CLI. Use this class when setting up agents for CLI usage.
+ * Use the base PluginService when you want a clean slate to register your own plugins.
+ */
+export class CliPluginService extends PluginService {
+  constructor(context: PluginContext) {
+    super(context);
+
+    // Register CLI default plugins
+    this.pluginMap.set("embeddings", new EmbeddingPlugin(context));
+    this.pluginMap.set("vim", new VimPlugin(context));
+    this.pluginMap.set("linter", new LinterPlugin(context));
+    this.pluginMap.set("language", new LanguagePlugin(context));
+    this.pluginMap.set("url", new UrlPlugin(context));
+    this.pluginMap.set("git", new GitPlugin(context));
+    this.pluginMap.set("tmux", new TmuxPlugin(context));
+    this.pluginMap.set("agents-md", new AgentsMdPlugin(context));
+    this.pluginMap.set("exec", new ExecPlugin(context));
+    this.pluginMap.set("skills", new SkillsPlugin(context));
+  }
+}

package/src/plugins/types.ts

@@ -1,5 +1,10 @@
 import { Embeddable, MinimalEmbedding } from "../types";
-import { AgentService, EventService, ToolsService } from "../services";
+import {
+  AgentService,
+  BehaviorsService,
+  EventService,
+  ToolsService,
+} from "../services";
 import { AIClient } from "../clients";
 import { PluginService } from "./plugins";
 
@@ -13,7 +18,7 @@ export interface PluginMeta {
 export interface Plugin {
   callMany(userInput?: string): Promise<string>;
   call(userInput?: string): Promise<string>;
-  embed(userInput?: string): Promise<MinimalEmbedding[]>;
+  embed?(userInput?: string): Promise<MinimalEmbedding[]>;
   enable(): void;
   disable(): void;
   isEnabled(): boolean;
@@ -27,4 +32,5 @@ export interface PluginContext {
   Clients?: AIClient;
   Tools?: ToolsService;
   Plugins?: PluginService;
+  Behaviors?: BehaviorsService;
 }