@tyvm/knowhow 0.0.158 → 0.0.160

package/src/generate.ts

@@ -1,529 +0,0 @@
-import { summarizeFiles, summarizeFile } from "./ai";
-import type { AgentOptions } from "./ai";
-import {
-  saveAllFileHashes,
-  getHashes,
-  checkNoFilesChanged,
-} from "./hashes";
-import * as fs from "fs";
-import * as path from "path";
-import * as crypto from "crypto";
-import { globSync } from "glob";
-
-import { GenerationSource } from "./types";
-import { readFile, writeFile } from "./utils";
-import { getConfig, loadPrompt } from "./config";
-import { services } from "./services/";
-import { convertToText } from "./conversion";
-
-// ---------------------------------------------------------------------------
-// Input pattern normalization
-// ---------------------------------------------------------------------------
-
-/**
- * Normalizes an input pattern to a valid glob pattern.
- * Supports:
- *   - Standard glob patterns (e.g. "src/**\/*.ts")
- *   - Brace expansion (e.g. "{src/a.ts,src/b.ts}")
- *   - Comma-separated file paths (e.g. "src/a.ts,src/b.ts") — auto-converted to brace expansion
- *   - Mixed comma-separated list with globs (e.g. "src/a.ts,src/commands/**\/*.ts")
- */
-export function normalizeInputPattern(input: string): string {
-  if (input.includes("{")) return input;
-  if (input.includes(",")) {
-    const parts = input.split(",").map((p) => p.trim());
-    return `{${parts.join(",")}}`;
-  }
-  return input;
-}
-
-// ---------------------------------------------------------------------------
-// GenerateOptions
-// ---------------------------------------------------------------------------
-
-export interface GenerateOptions {
-  /** Only run sources whose `name` matches exactly this value */
-  name?: string;
-  /** Only run sources whose name/input/output contains this substring */
-  filter?: string;
-  /**
-   * When true, agent-driven sources create .knowhow/processes/agents/<taskId>/
-   * so they appear in `knowhow agents list`. Overrides the per-source syncFs flag.
-   */
-  syncFs?: boolean;
-  /**
-   * When true, agent-driven sources push their work to a remote Knowhow task
-   * (identified by the source's taskId). Overrides the per-source syncRemote
-   * flag. Requires the sources to run as agents with a known taskId.
-   */
-  syncRemote?: boolean;
-  /** When true, skip hash checks and regenerate all matching sources regardless of changes */
-  force?: boolean;
-  /**
-   * Maximum number of generation sources to run in parallel within a single
-   * dependency wave. Defaults to 3. Set to 1 for fully sequential execution.
-   */
-  concurrency?: number;
-  /**
-   * In-memory generation sources. When provided, these are used INSTEAD of the
-   * sources defined in `.knowhow/knowhow.json`. This enables callers (e.g. an
-   * agent using the `runGenerate` tool) to define arbitrary pipelines at
-   * runtime without persisting them to config — supporting complex task
-   * orchestration with `dependsOn`, per-source `agent`, analysis, etc.
-   */
-  sources?: GenerationSource[];
-}
-
-// ---------------------------------------------------------------------------
-// Filter helper
-// ---------------------------------------------------------------------------
-
-/**
- * Returns true when a source should run given the provided filter options.
- */
-function sourceMatchesFilter(
-  source: GenerationSource,
-  options: GenerateOptions
-): boolean {
-  if (options.name) return source.name === options.name;
-  if (options.filter) {
-    const needle = options.filter.toLowerCase();
-    return [source.name, source.input, source.output]
-      .filter(Boolean)
-      .some((v) => v!.toLowerCase().includes(needle));
-  }
-  return true;
-}
-
-// ---------------------------------------------------------------------------
-// Dependency / concurrency helpers
-// ---------------------------------------------------------------------------
-
-/**
- * Detects whether sourceB's input references sourceA's output directory or file.
- *
- * Heuristic: if sourceA produces an output path and sourceB reads from that
- * same path (or a sub-path of it) then B depends on A.
- */
-function ioOverlaps(sourceA: GenerationSource, sourceB: GenerationSource): boolean {
-  const aOut = sourceA.output;
-  if (!aOut) return false;
-
-  const bInputPattern = normalizeInputPattern(sourceB.input || "");
-
-  // Strip brace expansion to individual paths for comparison
-  const bPaths = bInputPattern.startsWith("{")
-    ? bInputPattern.slice(1, -1).split(",").map((p) => p.trim())
-    : [bInputPattern];
-
-  for (const bPath of bPaths) {
-    // If aOut is a directory (ends with /) check if bPath starts with it
-    if (aOut.endsWith("/") && bPath.startsWith(aOut)) return true;
-    // If aOut is a file, check if bPath starts with aOut's directory
-    const aDir = aOut.endsWith("/") ? aOut : path.dirname(aOut) + "/";
-    if (bPath.startsWith(aDir)) return true;
-    // Direct prefix/glob match: e.g. aOut = "autodoc/" bPath = "autodoc/**/*.md"
-    if (bPath.startsWith(aOut.replace(/\/$/, ""))) return true;
-  }
-  return false;
-}
-
-/**
- * Builds a topological dependency graph and returns sources grouped into
- * sequential "waves". All sources within a wave have no un-satisfied
- * dependencies and can run concurrently. Each subsequent wave waits for
- * the previous wave to complete.
- *
- * Dependency resolution:
- *  1. Explicit `dependsOn: string[]` — names listed there must finish first.
- *  2. Automatic I/O overlap detection — if A writes to a path that B reads,
- *     B is automatically placed after A.
- */
-export function buildWaves(sources: GenerationSource[]): GenerationSource[][] {
-  const nameIndex = new Map<string, GenerationSource>();
-  for (const s of sources) {
-    if (s.name) nameIndex.set(s.name, s);
-  }
-
-  // Build adjacency: deps[i] = set of indices that source[i] depends on
-  const n = sources.length;
-  const deps: Set<number>[] = Array.from({ length: n }, () => new Set());
-
-  for (let i = 0; i < n; i++) {
-    const b = sources[i];
-
-    // Explicit dependsOn
-    for (const depName of b.dependsOn ?? []) {
-      const depIdx = sources.findIndex((s) => s.name === depName);
-      if (depIdx !== -1 && depIdx !== i) deps[i].add(depIdx);
-    }
-
-    // Auto I/O overlap
-    for (let j = 0; j < n; j++) {
-      if (i === j) continue;
-      if (ioOverlaps(sources[j], b)) deps[i].add(j);
-    }
-  }
-
-  // Kahn's algorithm to produce waves
-  const waves: GenerationSource[][] = [];
-  const completed = new Set<number>();
-
-  while (completed.size < n) {
-    const wave: number[] = [];
-    for (let i = 0; i < n; i++) {
-      if (completed.has(i)) continue;
-      const remaining = [...deps[i]].filter((d) => !completed.has(d));
-      if (remaining.length === 0) wave.push(i);
-    }
-
-    if (wave.length === 0) {
-      // Cycle detected — fall back to running everything remaining sequentially
-      console.warn(
-        "⚠️  Dependency cycle detected in generation sources — running remaining sources sequentially."
-      );
-      for (let i = 0; i < n; i++) {
-        if (!completed.has(i)) wave.push(i);
-      }
-    }
-
-    waves.push(wave.map((i) => sources[i]));
-    for (const i of wave) completed.add(i);
-  }
-
-  return waves;
-}
-
-/**
- * Runs up to `limit` async tasks concurrently, preserving order of
- * completion relative to input order.
- */
-async function runWithConcurrency<T>(
-  items: T[],
-  limit: number,
-  fn: (item: T, index: number) => Promise<void>
-): Promise<void> {
-  // Use a simple semaphore-style approach: keep up to `limit` promises running
-  // at the same time. As each finishes, the next item is started immediately.
-  const entries = [...items.entries()]; // [[0, item0], [1, item1], ...]
-  let cursor = 0;
-
-  async function worker(): Promise<void> {
-    while (cursor < entries.length) {
-      const [index, item] = entries[cursor++];
-      await fn(item, index);
-    }
-  }
-
-  // Spawn exactly `limit` workers (or fewer if there aren't enough items).
-  const workers: Promise<void>[] = [];
-  for (let i = 0; i < Math.min(limit, entries.length); i++) {
-    workers.push(worker());
-  }
-  await Promise.all(workers);
-}
-
-// ---------------------------------------------------------------------------
-// Output helpers
-// ---------------------------------------------------------------------------
-
-/**
- * Appends an instruction telling an assigned agent exactly which file to write
- * its generated output to.
- */
-export function withOutputTarget(prompt: string, outputFile: string): string {
-  return `${prompt}\n\n---\nGENERATION OUTPUT TARGET: Write your finished result to the file at \`${outputFile}\` using your file-writing tools (e.g. writeFileChunk). Create any parent directories as needed. Once the file is written with the complete, final content, call finalAnswer with a brief summary. Do NOT write to any other path.`;
-}
-
-/**
- * For agent-driven generation: if the agent already wrote the output file
- * itself, keep it. Otherwise, fall back to writing the returned content.
- */
-export async function writeAgentOrSummaryOutput(
-  outputFile: string,
-  summary: string,
-  agent?: string
-): Promise<void> {
-  const outputDir = path.dirname(outputFile);
-  if (outputDir && !fs.existsSync(outputDir)) {
-    fs.mkdirSync(outputDir, { recursive: true });
-  }
-
-  if (agent && fs.existsSync(outputFile)) {
-    console.log(
-      `Agent wrote output file directly: ${outputFile} (skipping summary write)`
-    );
-    return;
-  }
-
-  console.log("Writing summary to", outputFile);
-  await writeFile(outputFile, summary ?? "");
-}
-
-// ---------------------------------------------------------------------------
-// Multi-output generation
-// ---------------------------------------------------------------------------
-
-export async function handleMultiOutputGeneration(
-  model: string,
-  inputPattern: string,
-  files: string[],
-  prompt: string,
-  output: string,
-  outputExt = "mdx",
-  outputName?: string,
-  kind?: string,
-  agent?: string,
-  agentOpts?: AgentOptions,
-  force?: boolean
-) {
-  const promptHash = crypto.createHash("md5").update(prompt).digest("hex");
-  const hashes = await getHashes();
-
-  const inputPath = inputPattern.includes("**")
-    ? inputPattern.split("**")[0]
-    : "";
-
-  for (const file of files) {
-    const fileContent = await convertToText(file);
-    const fileHash = crypto.createHash("md5").update(fileContent).digest("hex");
-
-    if (!hashes[file]) {
-      hashes[file] = { promptHash: "", fileHash: "" };
-    }
-
-    const { name, ext, dir } = path.parse(file);
-    const nestedFolder = inputPath ? (dir + "/").replace(inputPath, "") : "";
-    const outputFolder = path.join(output, nestedFolder);
-
-    if (!fs.existsSync(outputFolder)) {
-      fs.mkdirSync(outputFolder, { recursive: true });
-    }
-
-    const outputFileName = outputName || name;
-    const outputFile = path.join(outputFolder, outputFileName + "." + outputExt);
-    console.log({ dir, inputPath, nestedFolder, outputFile });
-
-    const toCheck = [file, outputFile];
-    const noChanges = await checkNoFilesChanged(toCheck, promptHash, hashes);
-    if (noChanges && !force) {
-      console.log("Skipping file", file, "because it hasn't changed");
-      continue;
-    }
-
-    console.log("Summarizing", file);
-    const effectivePrompt =
-      prompt && agent ? withOutputTarget(prompt, outputFile) : prompt;
-    const summary = prompt
-      ? await summarizeFile(file, effectivePrompt, model, agent, agentOpts)
-      : fileContent;
-
-    await writeAgentOrSummaryOutput(outputFile, summary, agent);
-    await saveAllFileHashes(toCheck, promptHash);
-  }
-}
-
-// ---------------------------------------------------------------------------
-// Single-output generation
-// ---------------------------------------------------------------------------
-
-export async function handleSingleOutputGeneration(
-  model: string,
-  files: string[],
-  prompt: string,
-  outputFile: string,
-  kind?: string,
-  agent?: string,
-  agentOpts?: AgentOptions,
-  force?: boolean
-) {
-  const hashes = await getHashes();
-  const promptHash = crypto.createHash("md5").update(prompt).digest("hex");
-
-  const filesToCheck = [outputFile, ...files];
-  const noChanges = await checkNoFilesChanged(filesToCheck, promptHash, hashes);
-  if (noChanges && !force) {
-    console.log(`Skipping ${files.length} files because they haven't changed`);
-    return;
-  }
-
-  console.log("Summarizing", files.length, "files");
-  const outputDir = path.dirname(outputFile);
-  if (outputDir && !fs.existsSync(outputDir)) {
-    fs.mkdirSync(outputDir, { recursive: true });
-  }
-
-  const effectivePrompt =
-    prompt && agent ? withOutputTarget(prompt, outputFile) : prompt;
-
-  const summary = prompt
-    ? await summarizeFiles(files, effectivePrompt, model, agent, agentOpts)
-    : (await Promise.all(files.map(convertToText))).join("\n\n");
-
-  await writeAgentOrSummaryOutput(outputFile, summary, agent);
-  await saveAllFileHashes(filesToCheck, promptHash);
-}
-
-// ---------------------------------------------------------------------------
-// All-kinds generation (plugin + file fallback)
-// ---------------------------------------------------------------------------
-
-async function handleAllKindsGeneration(source: GenerationSource) {
-  const { Plugins } = services();
-  const { kind, input } = source;
-  if (Plugins.isPlugin(kind)) {
-    const data = await Plugins.call(kind, input);
-    if (source.output.endsWith("/")) {
-      throw new Error(`Plugin ${kind} can only output to a single file`);
-    }
-    const pluginOutputDir = path.dirname(source.output);
-    if (pluginOutputDir && !fs.existsSync(pluginOutputDir)) {
-      fs.mkdirSync(pluginOutputDir, { recursive: true });
-    }
-    await writeFile(source.output, data);
-  }
-  return handleFileKindGeneration(source);
-}
-
-async function handleFileKindGeneration(
-  source: GenerationSource,
-  agentOpts?: AgentOptions
-) {
-  const prompt = await loadPrompt(source.prompt);
-  const files = globSync(normalizeInputPattern(source.input));
-  console.log("Analyzing files: ", files);
-
-  if (source.output.endsWith("/")) {
-    await handleMultiOutputGeneration(
-      source.model,
-      source.input,
-      files,
-      prompt,
-      source.output,
-      source.outputExt,
-      source.outputName,
-      source.kind,
-      source.agent,
-      agentOpts
-    );
-  } else {
-    await handleSingleOutputGeneration(
-      source.model,
-      files,
-      prompt,
-      source.output,
-      source.kind,
-      source.agent,
-      agentOpts
-    );
-  }
-}
-
-// ---------------------------------------------------------------------------
-// Run a single source
-// ---------------------------------------------------------------------------
-
-async function runSource(
-  source: GenerationSource,
-  options: GenerateOptions
-): Promise<void> {
-  console.log("Generating", source.input, "to", source.output);
-
-  const effectiveSyncFs = options.syncFs ?? source.syncFs;
-  const effectiveSyncRemote = options.syncRemote ?? source.syncRemote;
-  const agentOpts: AgentOptions | undefined = source.agent
-    ? {
-        syncFs: effectiveSyncFs,
-        syncRemote: effectiveSyncRemote,
-        taskId:
-          source.taskId ??
-          (effectiveSyncFs || effectiveSyncRemote
-            ? `generate:${source.output.replace(/[^a-zA-Z0-9_.-]/g, "_")}`
-            : undefined),
-        maxTimeLimit: source.maxTimeLimit,
-        maxSpendLimit: source.maxSpendLimit,
-      }
-    : undefined;
-
-  if (source.kind === "file" || !source.kind) {
-    const files = globSync(normalizeInputPattern(source.input));
-    const prompt = await loadPrompt(source.prompt);
-
-    if (source.output.endsWith("/")) {
-      await handleMultiOutputGeneration(
-        source.model,
-        source.input,
-        files,
-        prompt,
-        source.output,
-        source.outputExt,
-        source.outputName,
-        source.kind,
-        source.agent,
-        agentOpts,
-        options.force
-      );
-    } else {
-      await handleSingleOutputGeneration(
-        source.model,
-        files,
-        prompt,
-        source.output,
-        source.kind,
-        source.agent,
-        agentOpts,
-        options.force
-      );
-    }
-  } else {
-    await handleAllKindsGeneration(source);
-  }
-}
-
-// ---------------------------------------------------------------------------
-// Main generate entry point
-// ---------------------------------------------------------------------------
-
-export async function generate(options: GenerateOptions = {}): Promise<void> {
-  let allSources: GenerationSource[];
-  if (options.sources && options.sources.length > 0) {
-    // In-memory pipeline definition — bypass config entirely.
-    allSources = options.sources;
-  } else {
-    const config = await getConfig();
-    allSources = config.sources ?? [];
-  }
-  const sources = allSources.filter((s) => sourceMatchesFilter(s, options));
-
-  if ((options.name || options.filter) && sources.length === 0) {
-    console.warn(
-      `No generation sources matched ${
-        options.name ? `name="${options.name}"` : `filter="${options.filter}"`
-      }. Available names: ${allSources
-        .map((s) => s.name)
-        .filter(Boolean)
-        .join(", ") || "(none defined)"}`
-    );
-    return;
-  }
-
-  const concurrency = options.concurrency ?? 3;
-  const waves = buildWaves(sources);
-
-  for (let wi = 0; wi < waves.length; wi++) {
-    const wave = waves[wi];
-    if (waves.length > 1) {
-      console.log(
-        `\n[Wave ${wi + 1}/${waves.length}] Running ${wave.length} source${wave.length !== 1 ? "s" : ""} (concurrency: ${concurrency})...`
-      );
-    }
-
-    await runWithConcurrency(wave, concurrency, (source) =>
-      runSource(source, options)
-    );
-  }
-
-  if (waves.length > 1) {
-    console.log(`\n✅ Generation complete (${waves.length} wave${waves.length !== 1 ? "s" : ""}, ${sources.length} sources).`);
-  }
-}

package/src/hashes.ts

@@ -1,180 +0,0 @@
-import fs from "fs";
-import * as crypto from "crypto";
-import { Hashes } from "./types";
-import { readFile } from "./utils";
-import { convertToText } from "./conversion";
-
-export async function getHashes() {
-  try {
-    const hashes = JSON.parse(await readFile(".knowhow/.hashes.json", "utf8"));
-    return hashes as Hashes;
-  } catch (err: any) {
-    if (err.code === "ENOENT") {
-      return {} as Hashes;
-    }
-    throw err;
-  }
-}
-
-/**
- * Atomically save hashes to disk — writes to a temp file then renames,
- * preventing concurrent writes from producing corrupted/truncated JSON.
- * The hash directory is optional state and may not exist in a fresh checkout.
- */
-export async function saveHashes(hashes: any) {
-  const target = ".knowhow/.hashes.json";
-  const tmp = `${target}.tmp.${process.pid}`;
-  try {
-    fs.mkdirSync(".knowhow", { recursive: true });
-    fs.writeFileSync(tmp, JSON.stringify(hashes, null, 2));
-    fs.renameSync(tmp, target);
-  } catch (err) {
-    try { fs.unlinkSync(tmp); } catch (_) {}
-    throw err;
-  }
-}
-
-export async function md5Hash(str: string) {
-  return crypto.createHash("md5").update(str).digest("hex");
-}
-
-export async function checkNoFilesChanged(
-  files: string[],
-  promptHash: string,
-  hashes: any
-) {
-  for (const file of files) {
-    if (!fs.existsSync(file)) {
-      return false;
-    }
-
-    // get the hash of the file
-    const fileContent = await convertToText(file);
-    const fileHash = crypto.createHash("md5").update(fileContent).digest("hex");
-
-    if (!hashes[file]) {
-      return false;
-    }
-
-    // Check if this file has changed (either format)
-    const matchesLegacy =
-      hashes[file].promptHash === promptHash &&
-      hashes[file].fileHash === fileHash;
-    const matchesCurrent = hashes[file][promptHash] === fileHash;
-
-    if (!matchesLegacy && !matchesCurrent) {
-      // This file has changed — re-generation needed
-      return false;
-    }
-    // This file is unchanged — continue checking the rest
-  }
-
-  return true;
-}
-
-export async function saveAllFileHashes(files: string[], promptHash: string) {
-  const hashes = await getHashes();
-
-  for (const file of files) {
-    const fileContent = await convertToText(file);
-    const fileHash = crypto.createHash("md5").update(fileContent).digest("hex");
-
-    if (!hashes[file]) {
-      hashes[file] = {
-        fileHash,
-        promptHash,
-      };
-    }
-    hashes[file][promptHash] = fileHash;
-  }
-
-  await saveHashes(hashes);
-}
-
-const UPLOAD_KEY = "upload";
-
-const DOWNLOAD_KEY = "download";
-
-/**
- * Returns true if the file has changed since the last successful upload
- * (or if it has never been uploaded before)
- */
-export async function hasFileChangedSinceUpload(
-  localPath: string,
-  hashes: any
-): Promise<boolean> {
-  if (!fs.existsSync(localPath)) return true;
-  const content = fs.readFileSync(localPath);
-  const currentHash = crypto.createHash("md5").update(content).digest("hex");
-  return hashes[localPath]?.[UPLOAD_KEY] !== currentHash;
-}
-
-/**
- * Mutates the provided hashes object with the upload hash for localPath.
- * If no hashes object is provided, loads, mutates, and saves independently.
- */
-export async function saveUploadHash(localPath: string, hashes?: any) {
-  const standalone = !hashes;
-  if (standalone) hashes = await getHashes();
-  const content = fs.readFileSync(localPath);
-  const currentHash = crypto.createHash("md5").update(content).digest("hex");
-  if (!hashes[localPath]) {
-    hashes[localPath] = { fileHash: currentHash, promptHash: "" };
-  }
-  hashes[localPath][UPLOAD_KEY] = currentHash;
-  if (standalone) await saveHashes(hashes);
-}
-
-/**
- * Returns true if the local file already matches the hash stored from
- * the last successful download, meaning we can skip the download.
- */
-export async function isLocalFileMatchingDownloadHash(
-  localPath: string,
-  hashes: any
-): Promise<boolean> {
-  if (!fs.existsSync(localPath)) return false;
-  const storedHash = hashes[localPath]?.[DOWNLOAD_KEY];
-  if (!storedHash) return false;
-  const content = fs.readFileSync(localPath);
-  const currentHash = crypto.createHash("sha256").update(content).digest("base64");
-  return storedHash === currentHash;
-}
-
-/**
- * Mutates the provided hashes object with the download hash for localPath.
- * If no hashes object is provided, loads, mutates, and saves independently.
- */
-export async function saveDownloadHash(localPath: string, hashes?: any) {
-  const standalone = !hashes;
-  if (standalone) hashes = await getHashes();
-  const content = fs.readFileSync(localPath);
-  const currentHash = crypto.createHash("sha256").update(content).digest("base64");
-  if (!hashes[localPath]) {
-    hashes[localPath] = { fileHash: currentHash, promptHash: "" };
-  }
-  hashes[localPath][DOWNLOAD_KEY] = currentHash;
-  if (standalone) await saveHashes(hashes);
-}
-
-/**
- * Compute SHA-256 of a local file, returned as base64 (matches S3 encoding)
- */
-export function computeSHA256Base64(filePath: string): string {
-  const content = fs.readFileSync(filePath);
-  return crypto.createHash("sha256").update(content).digest("base64");
-}
-
-/**
- * Returns true if the local file's SHA-256 matches the remote checksum,
- * meaning the file is up-to-date and download can be skipped.
- */
-export function isLocalFileMatchingRemote(
-  localPath: string,
-  remoteChecksumSHA256: string | null
-): boolean {
-  if (!remoteChecksumSHA256) return false;
-  if (!fs.existsSync(localPath)) return false;
-  const localChecksum = computeSHA256Base64(localPath);
-  return localChecksum === remoteChecksumSHA256;
-}