@stupify/cli 0.0.2 → 0.0.4

package/src/search-profile.ts

@@ -0,0 +1,89 @@
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { defaultChecks, searchChecks } from "./checks.ts";
+import type {
+  RepomixSearchConfig,
+  SearchProfile,
+  SearchProfilePattern,
+  StupifyCheck,
+} from "./types.ts";
+
+export async function loadSearchProfile(profilePath: string | null): Promise<SearchProfile | null> {
+  if (!profilePath) return null;
+  const fullPath = path.resolve(profilePath);
+  const value = JSON.parse(await readFile(fullPath, "utf8")) as SearchProfile;
+  if (!value || typeof value !== "object" || typeof value.id !== "string" || value.id.length === 0) {
+    throw new Error(`Invalid search profile: ${profilePath}`);
+  }
+  return value;
+}
+
+export function effectiveSearchChecks(
+  explicitCheckIds: readonly string[] | null,
+  profile: SearchProfile | null,
+): readonly StupifyCheck[] {
+  const checks = explicitCheckIds
+    ? searchChecks(explicitCheckIds)
+    : profilePatternIds(profile).length > 0
+      ? checksById(profilePatternIds(profile))
+      : searchChecks(null);
+
+  if (!profile?.patterns) return checks;
+  return checks.map((check) => applyPatternOverride(check, profile.patterns?.[check.id]));
+}
+
+export function effectiveMaxCandidates(defaultValue: number, profile: SearchProfile | null): number {
+  return positiveInteger(profile?.maxCandidates) ?? defaultValue;
+}
+
+export function effectiveMaxSearchInputTokens(defaultValue: number, profile: SearchProfile | null): number {
+  return positiveInteger(profile?.maxSearchInputTokens) ?? defaultValue;
+}
+
+export function effectiveRepomixConfig(
+  defaultValue: RepomixSearchConfig,
+  profile: SearchProfile | null,
+): RepomixSearchConfig {
+  const override = profile?.repomix;
+  if (!override) return defaultValue;
+  return {
+    compress: override.compress ?? defaultValue.compress,
+    showLineNumbers: override.showLineNumbers ?? defaultValue.showLineNumbers,
+    removeEmptyLines: override.removeEmptyLines ?? defaultValue.removeEmptyLines,
+    maxFileSizeBytes: positiveInteger(override.maxFileBytes) ?? defaultValue.maxFileSizeBytes,
+    maxTotalSizeBytes: positiveInteger(override.maxTotalBytes) ?? defaultValue.maxTotalSizeBytes,
+    ignorePatterns: override.ignorePatterns ?? defaultValue.ignorePatterns,
+  };
+}
+
+function profilePatternIds(profile: SearchProfile | null): readonly string[] {
+  if (!profile?.patterns) return [];
+  return Object.entries(profile.patterns)
+    .filter(([, pattern]) => pattern.enabled === true)
+    .map(([id]) => id);
+}
+
+function checksById(ids: readonly string[]): readonly StupifyCheck[] {
+  const byId = new Map<string, StupifyCheck>(defaultChecks.map((check) => [check.id, check]));
+  return ids.map((id) => {
+    const check = byId.get(id);
+    if (!check) throw new Error(`Unknown check in search profile: ${id}`);
+    return check;
+  });
+}
+
+function applyPatternOverride(check: StupifyCheck, override: SearchProfilePattern | undefined): StupifyCheck {
+  if (!override) return check;
+  return {
+    ...check,
+    searchPrompt: override.searchPrompt ?? check.searchPrompt,
+    searchExamples: {
+      match: override.matchExamples ?? check.searchExamples?.match ?? check.examples?.match ?? [],
+      nonMatch: override.nonMatchExamples ?? check.searchExamples?.nonMatch ?? check.examples?.noMatch ?? [],
+    },
+  };
+}
+
+function positiveInteger(value: unknown): number | null {
+  return Number.isInteger(value) && Number(value) > 0 ? Number(value) : null;
+}

package/src/sem-provider.ts

@@ -0,0 +1,282 @@
+import { execFile, spawn } from "node:child_process";
+import { mkdtemp, realpath, rm } from "node:fs/promises";
+import { createRequire } from "node:module";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { promisify } from "node:util";
+import { cachedJson, fingerprint } from "./cache.ts";
+import { readDiffFromStdin } from "./diff.ts";
+import {
+  sourceRangeForCommit,
+  sourceRangeForRecentCommits,
+  sourceRangeSince,
+  stagedDiff,
+} from "./git.ts";
+import type {
+  SearchCommand,
+  SemChange,
+  SemChangeSet,
+  SemChangeSummary,
+  SourceRange,
+} from "./types.ts";
+import { sourceId } from "./types.ts";
+
+const execFileAsync = promisify(execFile);
+
+export async function semChangeSetForCommand(
+  command: SearchCommand,
+): Promise<SemChangeSet> {
+  if (command.kind === "stdin") return semChangeSetFromPatch(await readDiffFromStdin(), command.debugSem);
+  if (command.kind === "staged") {
+    const diff = await stagedDiff();
+    if (!diff.text.trim()) return emptyChangeSet("staged", diff.stats);
+    return semChangeSetFromPatch(diff.text, command.debugSem, "staged");
+  }
+  if (command.kind === "commit") {
+    const range = await sourceRangeForCommit(command.commit);
+    const raw = await cachedSemDiff(
+      ["diff", "--commit", command.commit, "--format", "json"],
+      range,
+      command.debugSem,
+    );
+    return withContextWorkspace(normalizeSemDiff(raw, range), command.debugSem);
+  }
+
+  const range = await semRangeForCommand(command);
+  const raw = await cachedSemDiff(
+    ["diff", "--from", range.base, "--to", range.target, "--format", "json"],
+    range,
+    command.debugSem,
+  );
+  return withContextWorkspace(normalizeSemDiff(raw, range), command.debugSem);
+}
+
+function emptyChangeSet(label: string, stats: SourceRange["stats"]): SemChangeSet {
+  return {
+    id: sourceId(label),
+    label,
+    base: label,
+    target: label,
+    contextCwd: process.cwd(),
+    cleanup: async () => undefined,
+    changes: [],
+    summary: {
+      added: stats.additions,
+      deleted: stats.deletions,
+      modified: 0,
+      moved: 0,
+      renamed: 0,
+      fileCount: stats.filesChanged,
+      total: 0,
+    },
+  };
+}
+
+async function semRangeForCommand(command: SearchCommand): Promise<SourceRange> {
+  if (command.kind === "since") return sourceRangeSince(command.since);
+  if (command.kind === "commit") return sourceRangeForCommit(command.commit);
+  if (command.kind === "commits") return sourceRangeForRecentCommits(command.count);
+  throw new Error("sem cannot resolve stdin as a git range.");
+}
+
+async function semChangeSetFromPatch(patch: string, debugSem: boolean, label = "stdin"): Promise<SemChangeSet> {
+  if (!patch.trim()) throw new Error("No diff received on stdin.");
+  const raw = await cachedJson(
+    "sem-diff",
+    fingerprint({
+      version: 1,
+      cwd: process.cwd(),
+      command: ["diff", "--patch", "--format", "json"],
+      patchHash: fingerprint(patch),
+    }),
+    () => runSemWithInput(["diff", "--patch", "--format", "json"], patch, debugSem),
+  );
+  return {
+    ...normalizeSemDiff(raw, {
+    id: sourceId(label),
+    label,
+    base: label,
+    target: label,
+    stats: { filesChanged: 0, additions: 0, deletions: 0 },
+    }),
+    contextCwd: process.cwd(),
+    cleanup: async () => undefined,
+  };
+}
+
+async function cachedSemDiff(
+  args: readonly string[],
+  range: SourceRange,
+  debugSem: boolean,
+): Promise<unknown> {
+  return cachedJson(
+    "sem-diff",
+    fingerprint({
+      version: 1,
+      cwd: process.cwd(),
+      args,
+      base: range.base,
+      target: range.target,
+    }),
+    () => runSem(args, debugSem),
+  );
+}
+
+async function withContextWorkspace(changeSet: SemChangeSet, debugSem: boolean): Promise<SemChangeSet> {
+  const tempDir = await realpath(await mkdtemp(path.join(tmpdir(), "stupify-sem-context-")));
+  let worktreeAdded = false;
+  try {
+    await git(["worktree", "add", "--detach", tempDir, changeSet.target], debugSem);
+    worktreeAdded = true;
+    return {
+      ...changeSet,
+      contextCwd: tempDir,
+      cleanup: async () => cleanupWorktree(tempDir, worktreeAdded, debugSem),
+    };
+  } catch (error) {
+    await cleanupWorktree(tempDir, worktreeAdded, debugSem);
+    throw error;
+  }
+}
+
+async function runSem(args: readonly string[], debugSem: boolean, cwd = process.cwd()): Promise<unknown> {
+  if (debugSem) console.error(`sem ${args.join(" ")}`);
+  const { command, commandArgs } = resolveSemCommand(args);
+  try {
+    const { stdout, stderr } = await execFileAsync(command, commandArgs, {
+      cwd,
+      maxBuffer: 128 * 1024 * 1024,
+    });
+    if (debugSem && stderr.trim()) console.error(stderr.trim());
+    return JSON.parse(stdout);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    throw new Error(`sem failed. Install @ataraxy-labs/sem and ensure its binary is downloaded.\n${message}`);
+  }
+}
+
+async function runSemWithInput(args: readonly string[], stdin: string, debugSem: boolean): Promise<unknown> {
+  if (debugSem) console.error(`sem ${args.join(" ")}`);
+  const { command, commandArgs } = resolveSemCommand(args);
+  return new Promise((resolve, reject) => {
+    const child = spawn(command, commandArgs, { stdio: ["pipe", "pipe", "pipe"] });
+    const stdout: Buffer[] = [];
+    const stderr: Buffer[] = [];
+    child.stdout.on("data", (chunk: Buffer) => stdout.push(chunk));
+    child.stderr.on("data", (chunk: Buffer) => stderr.push(chunk));
+    child.on("error", (error) => reject(error));
+    child.on("close", (code) => {
+      const stderrText = Buffer.concat(stderr).toString("utf8");
+      if (debugSem && stderrText.trim()) console.error(stderrText.trim());
+      if (code !== 0) {
+        reject(new Error(`sem failed with exit code ${code}${stderrText ? `: ${stderrText.trim()}` : ""}`));
+        return;
+      }
+      try {
+        resolve(JSON.parse(Buffer.concat(stdout).toString("utf8")));
+      } catch (error) {
+        reject(error);
+      }
+    });
+    child.stdin.end(stdin);
+  });
+}
+
+async function git(args: readonly string[], debugSem: boolean): Promise<void> {
+  if (debugSem) console.error(`git ${args.join(" ")}`);
+  await execFileAsync("git", [...args], { maxBuffer: 128 * 1024 * 1024 });
+}
+
+async function cleanupWorktree(tempDir: string, worktreeAdded: boolean, debugSem: boolean): Promise<void> {
+  if (!worktreeAdded) {
+    await rm(tempDir, { recursive: true, force: true });
+    return;
+  }
+  try {
+    await git(["worktree", "remove", "--force", tempDir], debugSem);
+  } catch {
+    await rm(tempDir, { recursive: true, force: true });
+    await git(["worktree", "prune"], debugSem).catch(() => undefined);
+  }
+}
+
+function resolveSemCommand(args: readonly string[]): Readonly<{ command: string; commandArgs: readonly string[] }> {
+  const packageBin = semPackageBin();
+  if (packageBin) return { command: process.execPath, commandArgs: [packageBin, ...args] };
+  return { command: "sem", commandArgs: args };
+}
+
+function semPackageBin(): string | null {
+  try {
+    const require = createRequire(import.meta.url);
+    return require.resolve("@ataraxy-labs/sem/bin/sem.js");
+  } catch {
+    return null;
+  }
+}
+
+function normalizeSemDiff(value: unknown, range: SourceRange): SemChangeSet {
+  if (!value || typeof value !== "object") throw new Error("sem returned invalid diff JSON.");
+  const record = value as Record<string, unknown>;
+  const changes = Array.isArray(record.changes) ? record.changes.flatMap(normalizeSemChange) : [];
+  const summary = normalizeSemSummary(record.summary, changes);
+  return {
+    id: range.id,
+    label: range.label,
+    base: range.base,
+    target: range.target,
+    contextCwd: process.cwd(),
+    cleanup: async () => undefined,
+    changes,
+    summary,
+  };
+}
+
+function normalizeSemChange(value: unknown): readonly SemChange[] {
+  if (!value || typeof value !== "object") return [];
+  const record = value as Record<string, unknown>;
+  const entityId = stringValue(record.entityId);
+  const entityName = stringValue(record.entityName);
+  const entityType = stringValue(record.entityType);
+  const filePath = stringValue(record.filePath);
+  const changeType = stringValue(record.changeType);
+  if (!entityId || !entityName || !entityType || !filePath || !changeType) return [];
+  return [{
+    entityId,
+    entityName,
+    entityType,
+    filePath,
+    changeType,
+    beforeContent: nullableString(record.beforeContent),
+    afterContent: nullableString(record.afterContent),
+  }];
+}
+
+function normalizeSemSummary(value: unknown, changes: readonly SemChange[]): SemChangeSummary {
+  const record = value && typeof value === "object" ? value as Record<string, unknown> : {};
+  return {
+    added: numberValue(record.added) ?? countByChange(changes, "added"),
+    deleted: numberValue(record.deleted) ?? countByChange(changes, "deleted"),
+    modified: numberValue(record.modified) ?? countByChange(changes, "modified"),
+    moved: numberValue(record.moved) ?? countByChange(changes, "moved"),
+    renamed: numberValue(record.renamed) ?? countByChange(changes, "renamed"),
+    fileCount: numberValue(record.fileCount) ?? new Set(changes.map((change) => change.filePath)).size,
+    total: numberValue(record.total) ?? changes.length,
+  };
+}
+
+function countByChange(changes: readonly SemChange[], changeType: string): number {
+  return changes.filter((change) => change.changeType === changeType).length;
+}
+
+function stringValue(value: unknown): string {
+  return typeof value === "string" ? value : "";
+}
+
+function nullableString(value: unknown): string | null {
+  return typeof value === "string" ? value : null;
+}
+
+function numberValue(value: unknown): number | null {
+  return typeof value === "number" && Number.isFinite(value) ? value : null;
+}

package/src/stupify.ts

@@ -0,0 +1,285 @@
+#!/usr/bin/env node
+
+import { fileURLToPath } from "node:url";
+import { countPromptTokens, runSearch, searchRequest } from "./analysis.ts";
+import { searchChecks } from "./checks.ts";
+import { parseCommand } from "./command.ts";
+import { counterScoutTargets } from "./counter-scout.ts";
+import { runDoctor } from "./doctor.ts";
+import { runHookCommand } from "./hooks.ts";
+import { firstRunModelBootstrap, loadLocalModel } from "./model.ts";
+import { entityContextsFromChanges, emptyContextPack, repomixContextPack, repomixSearchConfig } from "./repomix-provider.ts";
+import { helpText, renderSearchRun } from "./render.ts";
+import {
+  effectiveMaxCandidates,
+  effectiveMaxSearchInputTokens,
+  effectiveRepomixConfig,
+  effectiveSearchChecks,
+  loadSearchProfile,
+} from "./search-profile.ts";
+import { semChangeSetForCommand } from "./sem-provider.ts";
+import { createTracer } from "./trace.ts";
+import type { SearchCommand, SearchProfile, SearchRunJson, SemContext, SemContextPack, StupifyCheck } from "./types.ts";
+
+export async function main(argv = process.argv.slice(2)): Promise<number> {
+  const startedAt = Date.now();
+  try {
+    const command = parseCommand(argv);
+    if (command.kind === "help") {
+      console.log(helpText());
+      return 0;
+    }
+    if (command.kind === "hook") {
+      console.log(await runHookCommand(command.action));
+      return 0;
+    }
+    if (command.kind === "doctor") {
+      const result = await runDoctor();
+      console.log(result.text);
+      return result.exitCode;
+    }
+    if (command.kind === "bench-search") {
+      const { runSearchBench } = await import("./search-bench.ts");
+      console.log(await runSearchBench(command.configPath));
+      return 0;
+    }
+
+    const run = await runSearchCommand(command, startedAt);
+    console.log(renderSearchRun(run, command));
+    return 0;
+  } catch (error) {
+    console.error(error instanceof Error ? error.message : String(error));
+    return 1;
+  }
+}
+
+export async function runSearchCommand(command: SearchCommand, startedAt: number): Promise<SearchRunJson> {
+  const t = createTracer({
+    writeLine: () => undefined,
+    onEvent: (event) => {
+      const parts = [`trace ${event.name}`, `${event.ms}ms`];
+      if (event.count !== undefined) parts.push(`count=${event.count}`);
+      if (event.detail) parts.push(event.detail);
+      console.error(parts.join(" "));
+    },
+  });
+
+  const profile = await loadSearchProfile(command.searchProfilePath);
+  const checks = profile ? effectiveSearchChecks(command.checkIds, profile) : searchChecks(command.checkIds);
+  const patternIds = checks.map((check) => check.id);
+  const maxCandidates = effectiveMaxCandidates(command.maxCandidates, profile);
+  const maxSearchInputTokens = effectiveMaxSearchInputTokens(command.maxSearchInputTokens, profile);
+  const { value: changeSet } = await t.trace(
+    "entity.diff",
+    () => semChangeSetForCommand(command),
+    {
+      count: (v) => v.summary.total,
+      detail: (v) => `${v.summary.fileCount} files`,
+    },
+  );
+
+  try {
+    printRunPlan(command, changeSet.summary.fileCount, changeSet.summary.total, patternIds);
+    const candidates = counterScoutTargets(changeSet, checks, maxCandidates);
+    const contexts = entityContextsFromChanges(candidates, changeSet.changes);
+    const targetsByPattern = countTargetsByPattern(contexts);
+    const targetsPreview = previewTargets(contexts);
+    if (contexts.length === 0) {
+      return {
+        schemaVersion: "search.v1",
+        mode: "search",
+        source: command.source,
+        model: { id: command.model },
+        patterns: patternIds,
+        stats: {
+          elapsedMs: Date.now() - startedAt,
+          modelCalls: 0,
+          skipped: true,
+          skipReason: "no_candidates",
+          filesChanged: changeSet.summary.fileCount,
+          entitiesScanned: changeSet.summary.total,
+          candidates: 0,
+          searchTargets: 0,
+          repomixFiles: 0,
+          repomixTokens: 0,
+          profileId: profile?.id,
+          targetsByPattern,
+          targetsPreview,
+        },
+        matches: [],
+      };
+    }
+
+    const baseRepomixConfig = effectiveRepomixConfig(repomixSearchConfig(), profile);
+    const initialPack = profile?.context === "sem"
+      ? emptyContextPack()
+      : await t.trace(
+        "context.pack",
+        () => repomixContextPack(changeSet.contextCwd, contexts, changeSet.changes, baseRepomixConfig),
+        {
+          count: (v) => v.filePaths.length,
+          detail: (v) => `${v.totalTokens} tokens`,
+        },
+      ).then((result) => result.value);
+    const packedFiles = new Set(initialPack.filePaths);
+    const searchContexts = profile?.context === "sem"
+      ? contexts
+      : contexts.filter((context) => context.filePath && packedFiles.has(context.filePath));
+    if (searchContexts.length === 0) {
+      return {
+        schemaVersion: "search.v1",
+        mode: "search",
+        source: command.source,
+        model: { id: command.model },
+        patterns: patternIds,
+        stats: {
+          elapsedMs: Date.now() - startedAt,
+          modelCalls: 0,
+          skipped: true,
+          skipReason: "no_candidates",
+          filesChanged: changeSet.summary.fileCount,
+          entitiesScanned: changeSet.summary.total,
+          candidates: contexts.length,
+          searchTargets: 0,
+          repomixFiles: initialPack.filePaths.length,
+          repomixTokens: initialPack.totalTokens,
+          repomixConfig: initialPack.config,
+          profileId: profile?.id,
+          targetsByPattern,
+          targetsPreview,
+        },
+        matches: [],
+      };
+    }
+    const pack = profile?.context === "sem" || searchContexts.length === contexts.length
+      ? initialPack
+      : await repomixContextPack(changeSet.contextCwd, searchContexts, changeSet.changes, baseRepomixConfig);
+
+    const modelPath = await firstRunModelBootstrap(command.model);
+    const model = await loadLocalModel(modelPath, command.model, "scout");
+    const request = buildSearchRequest(
+      changeSet,
+      searchContexts,
+      pack,
+      checks,
+      profile,
+      command.includeCounterReasonInPrompt,
+    );
+    const inputTokens = await countPromptTokens(model, request.prompt);
+    if (inputTokens > maxSearchInputTokens) {
+      return {
+        schemaVersion: "search.v1",
+        mode: "search",
+        source: command.source,
+        model: { id: command.model },
+        patterns: patternIds,
+        stats: {
+          elapsedMs: Date.now() - startedAt,
+          modelCalls: 0,
+          inputTokens,
+          inputTokenCap: maxSearchInputTokens,
+          skipped: true,
+          skipReason: "input_too_large",
+          filesChanged: changeSet.summary.fileCount,
+          entitiesScanned: changeSet.summary.total,
+          candidates: contexts.length,
+          searchTargets: searchContexts.length,
+          repomixFiles: pack.filePaths.length,
+          repomixTokens: pack.totalTokens,
+          repomixConfig: pack.config,
+          profileId: profile?.id,
+          targetsByPattern: countTargetsByPattern(searchContexts),
+          targetsPreview: previewTargets(searchContexts),
+        },
+        matches: [],
+      };
+    }
+
+    const { value: matches } = await t.trace(
+      "search.model",
+      () => runSearch(model, request),
+      { count: (v) => v.length },
+    );
+
+    return {
+      schemaVersion: "search.v1",
+      mode: "search",
+      source: command.source,
+      model: { id: command.model },
+      patterns: patternIds,
+      stats: {
+        elapsedMs: Date.now() - startedAt,
+        modelCalls: 1,
+        inputTokens,
+        inputTokenCap: maxSearchInputTokens,
+        filesChanged: changeSet.summary.fileCount,
+        entitiesScanned: changeSet.summary.total,
+        candidates: contexts.length,
+        searchTargets: searchContexts.length,
+        repomixFiles: pack.filePaths.length,
+        repomixTokens: pack.totalTokens,
+        repomixConfig: pack.config,
+        profileId: profile?.id,
+        targetsByPattern: countTargetsByPattern(searchContexts),
+        targetsPreview: previewTargets(searchContexts),
+      },
+      matches,
+    };
+  } finally {
+    await changeSet.cleanup();
+  }
+}
+
+function buildSearchRequest(
+  changeSet: Parameters<typeof searchRequest>[0]["changeSet"],
+  contexts: Parameters<typeof searchRequest>[0]["contexts"],
+  pack: SemContextPack,
+  patterns: readonly StupifyCheck[],
+  profile: SearchProfile | null,
+  includeCounterReasonInPrompt: boolean,
+) {
+  return searchRequest({
+    changeSet,
+    contexts,
+    pack,
+    patterns,
+    includeCounterReasonInPrompt: profile?.includeCounterReasonInPrompt ?? includeCounterReasonInPrompt,
+  });
+}
+
+function printRunPlan(
+  command: SearchCommand,
+  filesChanged: number,
+  entitiesScanned: number,
+  patternIds: readonly string[],
+): void {
+  if (command.json) return;
+  console.error("🧙 stupify 🪄");
+  console.error(`Mode: search (${command.source})`);
+  console.error(`Sem: ${filesChanged} files, ${entitiesScanned} changed entities`);
+  console.error(`Patterns: ${patternIds.join(", ")}`);
+}
+
+function countTargetsByPattern(contexts: readonly SemContext[]): Record<string, number> {
+  const counts: Record<string, number> = {};
+  for (const context of contexts) counts[context.checkId] = (counts[context.checkId] ?? 0) + 1;
+  return counts;
+}
+
+function previewTargets(contexts: readonly SemContext[]) {
+  return contexts.map((context) => ({
+    targetId: context.targetId,
+    patternId: context.checkId,
+    entityKind: context.entityKind || undefined,
+    sourceKind: context.filePath ? pathKind(context.filePath) : undefined,
+  }));
+}
+
+function pathKind(filePath: string): string {
+  const ext = filePath.split(".").pop();
+  return ext && ext !== filePath ? ext : "unknown";
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+  process.exitCode = await main();
+}