@tyvm/knowhow 0.0.130 → 0.0.132

package/src/commands/hash.ts

@@ -0,0 +1,170 @@
+import { Command } from "commander";
+import * as crypto from "crypto";
+import * as fs from "fs";
+import { execSync } from "child_process";
+import { globSync } from "glob";
+import { getHashes, saveHashes } from "../hashes";
+
+/**
+ * Compute an MD5 hash over the sorted contents of all matched files.
+ * Uses file content only (not mtime) for stability across CI clones.
+ */
+function hashFiles(files: string[]): string {
+  const hash = crypto.createHash("md5");
+  const sorted = [...files].sort();
+  for (const f of sorted) {
+    if (!fs.existsSync(f)) continue;
+    const content = fs.readFileSync(f);
+    hash.update(f);
+    hash.update(content);
+  }
+  return hash.digest("hex");
+}
+
+export function addHashCommand(program: Command): void {
+  program
+    .command("hash")
+    .description(
+      "Check whether a named set of input files has changed since last saved. " +
+        "Exits 0 (unchanged) or 1 (changed / first run). " +
+        "Use with shell short-circuit: (knowhow hash ... || run-build-step)"
+    )
+    .requiredOption(
+      "--name <name>",
+      "Unique name for this hash slot (e.g. 'build', 'generate-api'). Becomes the top-level key in .hashes.json."
+    )
+    .requiredOption(
+      "--input <glob>",
+      "Glob pattern(s) of input files to hash. Separate multiple globs with commas."
+    )
+    .option(
+      "--output <paths>",
+      "Optional output file(s) — if any are missing the check always returns changed (comma-separated)."
+    )
+    .option(
+      "--save",
+      "Save the current hash after a successful build step."
+    )
+    .option(
+      "--run <command>",
+      "Self-contained mode: if inputs are unchanged, skip and exit 0. " +
+        "If changed (or first run), run this shell command and — only if it " +
+        "succeeds — save the new hash. Replaces the '(hash) || (build && hash --save)' pattern."
+    )
+    .option("--verbose", "Print what files were found and the computed hash")
+    .action(
+      async (opts: {
+        name: string;
+        input: string;
+        output?: string;
+        save?: boolean;
+        run?: string;
+        verbose?: boolean;
+      }) => {
+        const hashes = await getHashes();
+
+        const inputGlobs = opts.input.split(",").map((g) => g.trim());
+        const outputPaths = opts.output
+          ? opts.output.split(",").map((p) => p.trim())
+          : [];
+
+        // Expand input globs
+        const inputFiles: string[] = [];
+        for (const pattern of inputGlobs) {
+          const matches = globSync(pattern, { nodir: true });
+          inputFiles.push(...matches);
+        }
+
+        if (opts.verbose) {
+          console.error(`[knowhow hash] name=${opts.name}`);
+          console.error(
+            `[knowhow hash] input files (${inputFiles.length}): ${inputFiles.slice(0, 10).join(", ")}${inputFiles.length > 10 ? "…" : ""}`
+          );
+          if (outputPaths.length) {
+            console.error(`[knowhow hash] output files: ${outputPaths.join(", ")}`);
+          }
+        }
+
+        const currentHash = hashFiles(inputFiles);
+
+        // --save mode: record into .hashes.json under hashes[name][inputGlob] = hash
+        if (opts.save) {
+          if (!hashes[opts.name]) hashes[opts.name] = {};
+          hashes[opts.name][opts.input] = currentHash;
+          await saveHashes(hashes);
+          if (opts.verbose) {
+            console.error(`[knowhow hash] saved hash ${currentHash} for "${opts.name}"`);
+          }
+          process.exit(0);
+        }
+
+        // Determine whether inputs (or missing outputs) indicate a rebuild is needed.
+        let changed = false;
+
+        // If any output file is missing, always report changed
+        for (const outPath of outputPaths) {
+          if (!fs.existsSync(outPath)) {
+            if (opts.verbose) {
+              console.error(`[knowhow hash] output file missing: ${outPath} → changed`);
+            }
+            changed = true;
+          }
+        }
+
+        const storedHash: string | undefined = hashes[opts.name]?.[opts.input];
+
+        if (opts.verbose) {
+          console.error(
+            `[knowhow hash] stored=${storedHash ?? "(none)"}  current=${currentHash}`
+          );
+        }
+
+        if (!(storedHash && storedHash === currentHash)) {
+          changed = true;
+        }
+
+        // --run mode: self-contained. Skip when unchanged; otherwise run the
+        // command and only save the new hash if it succeeds.
+        if (opts.run) {
+          if (!changed) {
+            if (opts.verbose) {
+              console.error(`[knowhow hash] ✓ unchanged — skipping "${opts.run}"`);
+            }
+            process.exit(0);
+          }
+
+          if (opts.verbose) {
+            console.error(`[knowhow hash] ✗ changed — running "${opts.run}"`);
+          }
+          try {
+            execSync(opts.run, { stdio: "inherit", shell: "/bin/sh" });
+          } catch (err: any) {
+            // Command failed — do NOT save the hash so it retries next time.
+            process.exit(typeof err?.status === "number" ? err.status : 1);
+          }
+
+          // Command succeeded — record the hash of the inputs that produced it.
+          if (!hashes[opts.name]) hashes[opts.name] = {};
+          hashes[opts.name][opts.input] = currentHash;
+          await saveHashes(hashes);
+          if (opts.verbose) {
+            console.error(`[knowhow hash] saved hash ${currentHash} for "${opts.name}"`);
+          }
+          process.exit(0);
+        }
+
+        // Legacy check mode: exit 0 (unchanged) or 1 (changed).
+        if (!changed) {
+          if (opts.verbose) {
+            console.error(`[knowhow hash] ✓ unchanged — skipping`);
+          }
+          process.exit(0);
+        }
+
+        if (opts.verbose) {
+          console.error(`[knowhow hash] ✗ changed — rebuild needed`);
+        }
+        process.exit(1);
+      }
+    );
+}

package/src/commands/misc.ts

@@ -2,7 +2,8 @@ import { Command } from "commander";
 import { execSync } from "child_process";
 import { version } from "../../package.json";
 import { logger } from "../logger";
-import { generate, embed, upload, download, purge } from "../index";
+import { embed, upload, download, purge } from "../index";
+import { generate, buildWaves, normalizeInputPattern, GenerateOptions } from "../generate";
 import { init } from "../config";
 import { login } from "../login";
 import { KnowhowSimpleClient } from "../services/KnowhowClient";
@@ -53,13 +54,166 @@ export function addGenerateCommand(program: Command): void {
   program
     .command("generate")
     .description("Generate documentation")
-    .action(async () => {
+    .option(
+      "--name <name>",
+      "Only run the generation source with this exact `name`"
+    )
+    .option(
+      "--filter <substring>",
+      "Only run generation sources whose name/input/output contains this substring"
+    )
+    .option(
+      "--sync-fs",
+      "Enable filesystem sync for agent-driven sources (creates .knowhow/processes/agents/<taskId>/ so tasks appear in `knowhow agents list`)"
+    )
+    .option(
+      "--force",
+      "Skip hash checks and regenerate all matching sources regardless of whether inputs/outputs have changed"
+    )
+    .option(
+      "--concurrency <n>",
+      "Maximum number of sources to run in parallel within a dependency wave (default: 3)",
+      (v) => parseInt(v, 10)
+    )
+    .option(
+      "--plan",
+      "Show the execution plan (dependency waves, skip/run status) without actually generating anything"
+    )
+    .action(async (options: {
+      name?: string;
+      filter?: string;
+      syncFs?: boolean;
+      force?: boolean;
+      concurrency?: number;
+      plan?: boolean;
+    }) => {
       const { setupServices } = await import("./services");
       await setupServices();
-      await generate();
+
+      if (options.plan) {
+        await showGeneratePlan(options);
+        return;
+      }
+
+      await generate({
+        name: options.name,
+        filter: options.filter,
+        syncFs: options.syncFs,
+        force: options.force,
+        concurrency: options.concurrency,
+      });
     });
 }
 
+async function showGeneratePlan(options: {
+  name?: string;
+  filter?: string;
+  force?: boolean;
+  concurrency?: number;
+}): Promise<void> {
+  const { getConfig } = await import("../config");
+  const { getHashes, checkNoFilesChanged } = await import("../hashes");
+  const { globSync } = await import("glob");
+  const crypto = await import("crypto");
+  const { loadPrompt } = await import("../config");
+
+  const config = await getConfig();
+  const allSources = config.sources ?? [];
+
+  // Filter sources
+  const sources = allSources.filter((s) => {
+    if (options.name) return s.name === options.name;
+    if (options.filter) {
+      const needle = options.filter.toLowerCase();
+      return [s.name, s.input, s.output].filter(Boolean).some((v) => v!.toLowerCase().includes(needle));
+    }
+    return true;
+  });
+
+  if (sources.length === 0) {
+    console.warn("No generation sources matched the filter.");
+    return;
+  }
+
+  const concurrency = options.concurrency ?? 3;
+  const waves = buildWaves(sources);
+  const hashes = await getHashes();
+
+  console.log(`\n📋 Generation Plan  (concurrency: ${concurrency})\n`);
+  console.log(`   Total sources : ${sources.length}`);
+  console.log(`   Waves         : ${waves.length}`);
+  console.log("");
+
+  for (let wi = 0; wi < waves.length; wi++) {
+    const wave = waves[wi];
+    console.log(`┌─ Wave ${wi + 1}/${waves.length}  (${wave.length} source${wave.length !== 1 ? "s" : ""} in parallel)`);
+
+    for (const source of wave) {
+      // Determine skip/run status via hash check
+      let status = "▶  run ";
+      let reason = "";
+
+      if (!options.force) {
+        try {
+          const prompt = await loadPrompt(source.prompt);
+          const promptHash = crypto.createHash("md5").update(prompt || "").digest("hex");
+          let filesToCheck: string[];
+
+          if (source.output.endsWith("/")) {
+            // For multi-output we check the input files individually;
+            // just check one representative glob match to indicate staleness
+            const files = globSync(normalizeInputPattern(source.input || ""));
+            filesToCheck = files.length > 0 ? [files[0], source.output] : [source.output];
+          } else {
+            const files = globSync(normalizeInputPattern(source.input || ""));
+            filesToCheck = [source.output, ...files];
+          }
+
+          const noChanges = await checkNoFilesChanged(filesToCheck, promptHash, hashes);
+          if (noChanges) {
+            status = "⏭  skip";
+            reason = " (no changes)";
+          }
+        } catch {
+          // If hash check fails just mark as run
+        }
+      } else {
+        reason = " (--force)";
+      }
+
+      const deps = source.dependsOn?.length ? `  [depends: ${source.dependsOn.join(", ")}]` : "";
+      const name = source.name ? `${source.name}` : "(unnamed)";
+      const arrow = `${source.input || "?"} → ${source.output || "?"}`;
+      console.log(`│  ${status}  ${name.padEnd(30)} ${arrow}${reason}${deps}`);
+    }
+    console.log("└" + "─".repeat(60));
+    console.log("");
+  }
+
+  const runCount = (await Promise.all(
+    waves.flat().map(async (source) => {
+      if (options.force) return true;
+      try {
+        const prompt = await loadPrompt(source.prompt);
+        const promptHash = crypto.createHash("md5").update(prompt || "").digest("hex");
+        const files = globSync(normalizeInputPattern(source.input || ""));
+        const filesToCheck = source.output.endsWith("/")
+          ? (files.length > 0 ? [files[0], source.output] : [source.output])
+          : [source.output, ...files];
+        const noChanges = await checkNoFilesChanged(filesToCheck, promptHash, hashes);
+        return !noChanges;
+      } catch {
+        return true;
+      }
+    })
+  )).filter(Boolean).length;
+
+  console.log(`   Would run  : ${runCount} source${runCount !== 1 ? "s" : ""}`);
+  console.log(`   Would skip : ${sources.length - runCount} source${sources.length - runCount !== 1 ? "s" : ""}`);
+  console.log("");
+  console.log("Run `knowhow generate` to execute, or add --force to skip hash checks.\n");
+}
+
 export function addEmbedCommands(program: Command): void {
   program
     .command("embed")

package/src/commands/skills.ts

@@ -68,4 +68,40 @@ export function addSkillsCommand(program: Command): void {
       const svc = new SkillsService({ global: opts.global });
       await svc.update(skillNames);
     });
+
+  skills
+    .command("upload")
+    .description(
+      "Upload locally installed skills from .knowhow/skills/ to the backend as behaviors (isSkill: true)"
+    )
+    .option("--global", "Upload from global ~/.knowhow/skills/")
+    .option(
+      "--install-dir <dir>",
+      "Custom install directory to upload from (overrides default and --global)"
+    )
+    .action(async (opts: { global?: boolean; installDir?: string }) => {
+      const svc = new SkillsService({
+        global: opts.global,
+        installDir: opts.installDir,
+      });
+      await svc.upload();
+    });
+
+  skills
+    .command("download")
+    .description(
+      "Download skill behaviors from the backend and write them as SKILL.md files into .knowhow/skills/<name>/SKILL.md"
+    )
+    .option("--global", "Download into global ~/.knowhow/skills/")
+    .option(
+      "--install-dir <dir>",
+      "Custom install directory (overrides default and --global)"
+    )
+    .action(async (opts: { global?: boolean; installDir?: string }) => {
+      const svc = new SkillsService({
+        global: opts.global,
+        installDir: opts.installDir,
+      });
+      await svc.download();
+    });
 }

package/src/config.ts

@@ -7,6 +7,9 @@ import { promisify } from "util";
 import { KNOWHOW_API_URL } from "./services/KnowhowClient";
 import {
   Config,
+  SourceRef,
+  GenerationSource,
+  EmbedSource,
   Language,
   AssistantConfig,
   Models,
@@ -270,11 +273,58 @@ export function getConfigSync() {
 }
 
 let loggedWarning = false;
+/**
+ * Checks whether an item in a sources/embedSources array is a SourceRef
+ * (i.e. `{ from: "..." }`) rather than a real source entry.
+ */
+function isSourceRef(item: unknown): item is SourceRef {
+  return (
+    typeof item === "object" &&
+    item !== null &&
+    "from" in item &&
+    typeof (item as any).from === "string"
+  );
+}
+
+/**
+ * Resolves `{ from: "./path/to/file.json" }` entries in `sources` and
+ * `embedSources` arrays. Each referenced file must contain a JSON array of
+ * source entries. Paths are resolved relative to `.knowhow/`.
+ */
+function resolveSourceRefs<T>(
+  items: (T | SourceRef)[] | undefined,
+  label: string
+): T[] {
+  if (!items) return [];
+  const resolved: T[] = [];
+  for (const item of items) {
+    if (isSourceRef(item)) {
+      const refPath = path.resolve(".knowhow", item.from);
+      try {
+        const raw = fs.readFileSync(refPath, "utf8");
+        const arr = JSON.parse(raw);
+        if (!Array.isArray(arr)) {
+          console.warn(`⚠️  ${label} ref "${item.from}" did not contain a JSON array — skipping.`);
+          continue;
+        }
+        resolved.push(...(arr as T[]));
+      } catch (e) {
+        console.warn(`⚠️  Could not load ${label} ref "${item.from}": ${(e as Error).message}`);
+      }
+    } else {
+      resolved.push(item as T);
+    }
+  }
+  return resolved;
+}
+
 export async function getConfig() {
   if (!fs.existsSync(".knowhow/knowhow.json")) {
     if (!loggedWarning) {
       loggedWarning = true;
-      if (!process.argv.includes("init")) {
+      const warnNeeds = process.env.KNOWHOW_WARN_NEEDS ?? "";
+      const shouldWarn = warnNeeds.split(",").includes("config");
+      if (shouldWarn && !process.argv.includes("init")) {
         console.warn(
           "KnowHow config file not found. Please run `knowhow init` to create it."
         );
@@ -285,7 +335,9 @@ export async function getConfig() {
   try {
     const config = await readFile(".knowhow/knowhow.json", "utf8");
     const parsedConfig = JSON.parse(config);
-
+    // Resolve any { from: "..." } SourceRef entries in sources / embedSources
+    parsedConfig.sources = resolveSourceRefs<GenerationSource>(parsedConfig.sources, "sources");
+    parsedConfig.embedSources = resolveSourceRefs<EmbedSource>(parsedConfig.embedSources, "embedSources");
     return parsedConfig as Config;
   } catch (error) {
     console.error("Error reading .knowhow/knowhow.json:", error);