@stupify/cli 0.0.6 → 0.0.7

package/dist/checks.js

@@ -173,7 +173,7 @@ Prefer no match over a weak match.`,
             "helper is domain-specific or used by multiple local call sites",
         ],
         hookMode: "warn",
-        searchPrompt: "Find only tiny generic utility functions that recreate common helpers such as clamp, debounce, throttle, slugify, group, sort, pick, omit, uniq, or shuffle without domain-specific behavior. Do not match resolve/parse/format helpers, domain formatting, feature constants, or helpers with multiple obvious call sites.",
+        searchPrompt: "Find only tiny generic utility functions that recreate common helpers such as clamp, debounce, throttle, slugify, sort, pick, omit, uniq, or shuffle without domain-specific behavior. Do not match group/resolve/parse/format helpers, domain formatting, feature constants, or helpers with multiple obvious call sites.",
         searchExamples: {
             match: [
                 "clampValue returns min, max, or value.",

package/dist/constants.d.ts

@@ -1,4 +1,4 @@
-export declare const VERSION = "0.0.6";
+export declare const VERSION = "0.0.7";
 import type { ModelConfig, ModelId } from "./types.ts";
 export declare const DEFAULT_MODEL_ID: ModelId;
 export declare const MODEL_REGISTRY: Record<ModelId, ModelConfig>;

package/dist/constants.js

@@ -1,4 +1,4 @@
-export const VERSION = "0.0.6";
+export const VERSION = "0.0.7";
 export const DEFAULT_MODEL_ID = "gemma-4-e2b";
 export const MODEL_REGISTRY = {
     "gemma-4-e2b": {

package/dist/counter-scout.js

@@ -120,7 +120,7 @@ function lintBypassSignal(value) {
 }
 function reinventedUtilitySignal(change) {
     const name = change.entityName;
-    if (!/^(clamp|debounce|throttle|slug|slugify|group|sort|shuffle|memoize|pick|omit|uniq)/i.test(name))
+    if (!/^(clamp|debounce|throttle|slug|slugify|sort|shuffle|memoize|pick|omit|uniq)/i.test(name))
         return false;
     const content = change.afterContent ?? "";
     if (/currency|invoice|refund|subscription|tier|domain/i.test(`${name}\n${content}`))

package/dist/stupify.js

@@ -133,9 +133,18 @@ export async function runSearchCommand(command, startedAt) {
         const pack = profile?.context === "sem" || searchContexts.length === contexts.length
             ? initialPack
             : await repomixContextPack(changeSet.contextCwd, searchContexts, changeSet.changes, baseRepomixConfig);
-        const request = buildSearchRequest(changeSet, searchContexts, pack, checks, profile, command.includeCounterReasonInPrompt);
-        const estimatedInputTokens = estimatePromptTokens(request.prompt);
-        if (estimatedInputTokens > maxSearchInputTokens) {
+        const batches = await buildSearchBatches({
+            command,
+            changeSet,
+            contexts: searchContexts,
+            initialPack: pack,
+            checks,
+            profile,
+            includeCounterReasonInPrompt: command.includeCounterReasonInPrompt,
+            maxSearchInputTokens,
+            baseRepomixConfig,
+        });
+        if (batches.batches.length === 0) {
             return {
                 schemaVersion: "search.v1",
                 mode: "search",
@@ -145,7 +154,7 @@ export async function runSearchCommand(command, startedAt) {
                 stats: {
                     elapsedMs: Date.now() - startedAt,
                     modelCalls: 0,
-                    inputTokens: estimatedInputTokens,
+                    inputTokens: batches.estimatedInputTokens,
                     inputTokenCap: maxSearchInputTokens,
                     skipped: true,
                     skipReason: "input_too_large",
@@ -156,6 +165,8 @@ export async function runSearchCommand(command, startedAt) {
                     repomixFiles: pack.filePaths.length,
                     repomixTokens: pack.totalTokens,
                     repomixConfig: pack.config,
+                    searchBatches: 0,
+                    skippedTargets: batches.skippedTargets,
                     profileId: profile?.id,
                     targetsByPattern: countTargetsByPattern(searchContexts),
                     targetsPreview: previewTargets(searchContexts),
@@ -163,38 +174,33 @@ export async function runSearchCommand(command, startedAt) {
                 matches: [],
             };
         }
+        if (batches.wasSplit && !command.json) {
+            console.error(`Search input is large; queued ${batches.batches.length} smaller search batches.`);
+            if (batches.skippedTargets > 0) {
+                console.error(`Skipped ${batches.skippedTargets} oversized targets that could not fit alone.`);
+            }
+        }
         const modelPath = await firstRunModelBootstrap(command.model);
         const model = await loadLocalModel(modelPath, command.model, "scout");
-        const inputTokens = await countPromptTokens(model, request.prompt);
-        if (inputTokens > maxSearchInputTokens) {
-            return {
-                schemaVersion: "search.v1",
-                mode: "search",
-                source: command.source,
-                model: { id: command.model },
-                patterns: patternIds,
-                stats: {
-                    elapsedMs: Date.now() - startedAt,
-                    modelCalls: 0,
-                    inputTokens,
-                    inputTokenCap: maxSearchInputTokens,
-                    skipped: true,
-                    skipReason: "input_too_large",
-                    filesChanged: changeSet.summary.fileCount,
-                    entitiesScanned: changeSet.summary.total,
-                    candidates: contexts.length,
-                    searchTargets: searchContexts.length,
-                    repomixFiles: pack.filePaths.length,
-                    repomixTokens: pack.totalTokens,
-                    repomixConfig: pack.config,
-                    profileId: profile?.id,
-                    targetsByPattern: countTargetsByPattern(searchContexts),
-                    targetsPreview: previewTargets(searchContexts),
-                },
-                matches: [],
-            };
+        const matches = [];
+        let modelCalls = 0;
+        let inputTokens = 0;
+        let exactSkippedTargets = batches.skippedTargets;
+        for (const batch of batches.batches) {
+            const batchInputTokens = await countPromptTokens(model, batch.request.prompt);
+            inputTokens += batchInputTokens;
+            if (batchInputTokens > maxSearchInputTokens) {
+                exactSkippedTargets += batch.contexts.length;
+                if (!command.json) {
+                    console.error(`Skipped ${batch.contexts.length} targets after exact token count exceeded the limit.`);
+                }
+                continue;
+            }
+            const { value } = await t.trace("search.model", () => runSearch(model, batch.request), { count: (v) => v.length });
+            modelCalls += 1;
+            matches.push(...value);
         }
-        const { value: matches } = await t.trace("search.model", () => runSearch(model, request), { count: (v) => v.length });
+        const uniqueMatches = dedupeMatches(matches);
         return {
             schemaVersion: "search.v1",
             mode: "search",
@@ -203,7 +209,7 @@ export async function runSearchCommand(command, startedAt) {
             patterns: patternIds,
             stats: {
                 elapsedMs: Date.now() - startedAt,
-                modelCalls: 1,
+                modelCalls,
                 inputTokens,
                 inputTokenCap: maxSearchInputTokens,
                 filesChanged: changeSet.summary.fileCount,
@@ -213,17 +219,91 @@ export async function runSearchCommand(command, startedAt) {
                 repomixFiles: pack.filePaths.length,
                 repomixTokens: pack.totalTokens,
                 repomixConfig: pack.config,
+                searchBatches: batches.batches.length,
+                skippedTargets: exactSkippedTargets,
                 profileId: profile?.id,
                 targetsByPattern: countTargetsByPattern(searchContexts),
                 targetsPreview: previewTargets(searchContexts),
             },
-            matches,
+            matches: uniqueMatches,
         };
     }
     finally {
         await changeSet.cleanup();
     }
 }
+function dedupeMatches(matches) {
+    const seen = new Set();
+    return matches.filter((match) => {
+        const key = `${match.patternId}\n${match.proof.trim()}`;
+        if (seen.has(key))
+            return false;
+        seen.add(key);
+        return true;
+    });
+}
+async function buildSearchBatches(input) {
+    const first = makeSearchBatch(input, input.contexts, input.initialPack);
+    if (first.estimatedInputTokens <= input.maxSearchInputTokens) {
+        return {
+            batches: [first],
+            estimatedInputTokens: first.estimatedInputTokens,
+            skippedTargets: 0,
+            wasSplit: false,
+        };
+    }
+    const batches = [];
+    let skippedTargets = 0;
+    let currentContexts = [];
+    let currentBatch = null;
+    for (const context of input.contexts) {
+        const candidateContexts = [...currentContexts, context];
+        const candidateBatch = await makeSearchBatchWithPack(input, candidateContexts);
+        if (candidateBatch.estimatedInputTokens <= input.maxSearchInputTokens) {
+            currentContexts = candidateContexts;
+            currentBatch = candidateBatch;
+            continue;
+        }
+        if (currentBatch) {
+            batches.push(currentBatch);
+            currentContexts = [];
+            currentBatch = null;
+        }
+        const singleBatch = candidateContexts.length === 1
+            ? candidateBatch
+            : await makeSearchBatchWithPack(input, [context]);
+        if (singleBatch.estimatedInputTokens <= input.maxSearchInputTokens) {
+            currentContexts = [context];
+            currentBatch = singleBatch;
+        }
+        else {
+            skippedTargets += 1;
+        }
+    }
+    if (currentBatch)
+        batches.push(currentBatch);
+    return {
+        batches,
+        estimatedInputTokens: first.estimatedInputTokens,
+        skippedTargets,
+        wasSplit: true,
+    };
+}
+function makeSearchBatch(input, contexts, pack) {
+    const request = buildSearchRequest(input.changeSet, contexts, pack, input.checks, input.profile, input.includeCounterReasonInPrompt);
+    return {
+        contexts,
+        pack,
+        request,
+        estimatedInputTokens: estimatePromptTokens(request.prompt),
+    };
+}
+async function makeSearchBatchWithPack(input, contexts) {
+    const pack = input.profile?.context === "sem"
+        ? emptyContextPack()
+        : await repomixContextPack(input.changeSet.contextCwd, contexts, input.changeSet.changes, input.baseRepomixConfig);
+    return makeSearchBatch(input, contexts, pack);
+}
 function buildSearchRequest(changeSet, contexts, pack, patterns, profile, includeCounterReasonInPrompt) {
     return searchRequest({
         changeSet,
@@ -261,7 +341,7 @@ function sourceLabel(command) {
     return "stdin diff";
 }
 function estimatePromptTokens(prompt) {
-    return Math.ceil(prompt.length / 4);
+    return Math.ceil(prompt.length / 3);
 }
 function countTargetsByPattern(contexts) {
     const counts = {};

package/dist/types.d.ts

@@ -219,6 +219,8 @@ export type SearchRunJson = Readonly<{
         repomixTokens?: number;
         repomixConfig?: RepomixSearchConfig;
         searchTargets?: number;
+        searchBatches?: number;
+        skippedTargets?: number;
         profileId?: string;
         targetsByPattern?: Readonly<Record<string, number>>;
         targetsPreview?: readonly SearchTargetPreview[];

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stupify/cli",
-  "version": "0.0.6",
+  "version": "0.0.7",
   "description": "Local-only diagnostic CLI for checking whether AI is making you dumber.",
   "private": false,
   "type": "module",

package/src/checks.ts

@@ -174,7 +174,7 @@ Prefer no match over a weak match.`,
       "helper is domain-specific or used by multiple local call sites",
     ],
     hookMode: "warn",
-    searchPrompt: "Find only tiny generic utility functions that recreate common helpers such as clamp, debounce, throttle, slugify, group, sort, pick, omit, uniq, or shuffle without domain-specific behavior. Do not match resolve/parse/format helpers, domain formatting, feature constants, or helpers with multiple obvious call sites.",
+    searchPrompt: "Find only tiny generic utility functions that recreate common helpers such as clamp, debounce, throttle, slugify, sort, pick, omit, uniq, or shuffle without domain-specific behavior. Do not match group/resolve/parse/format helpers, domain formatting, feature constants, or helpers with multiple obvious call sites.",
     searchExamples: {
       match: [
         "clampValue returns min, max, or value.",

package/src/constants.ts

@@ -1,4 +1,4 @@
-export const VERSION = "0.0.6";
+export const VERSION = "0.0.7";
 import type { ModelConfig, ModelId } from "./types.ts";
 
 export const DEFAULT_MODEL_ID: ModelId = "gemma-4-e2b";

package/src/counter-scout.ts

@@ -140,7 +140,7 @@ function lintBypassSignal(value: string): boolean {
 
 function reinventedUtilitySignal(change: SemChange): boolean {
   const name = change.entityName;
-  if (!/^(clamp|debounce|throttle|slug|slugify|group|sort|shuffle|memoize|pick|omit|uniq)/i.test(name)) return false;
+  if (!/^(clamp|debounce|throttle|slug|slugify|sort|shuffle|memoize|pick|omit|uniq)/i.test(name)) return false;
   const content = change.afterContent ?? "";
   if (/currency|invoice|refund|subscription|tier|domain/i.test(`${name}\n${content}`)) return false;
   return true;

package/src/stupify.ts

@@ -2,7 +2,7 @@
 
 import { realpathSync } from "node:fs";
 import { fileURLToPath } from "node:url";
-import { countPromptTokens, runSearch, searchRequest } from "./analysis.ts";
+import { countPromptTokens, runSearch, searchRequest, type SearchRequest } from "./analysis.ts";
 import { searchChecks } from "./checks.ts";
 import { parseCommand } from "./command.ts";
 import { counterScoutTargets } from "./counter-scout.ts";
@@ -153,17 +153,19 @@ export async function runSearchCommand(command: SearchCommand, startedAt: number
     const pack = profile?.context === "sem" || searchContexts.length === contexts.length
       ? initialPack
       : await repomixContextPack(changeSet.contextCwd, searchContexts, changeSet.changes, baseRepomixConfig);
-
-    const request = buildSearchRequest(
+    const batches = await buildSearchBatches({
+      command,
       changeSet,
-      searchContexts,
-      pack,
+      contexts: searchContexts,
+      initialPack: pack,
       checks,
       profile,
-      command.includeCounterReasonInPrompt,
-    );
-    const estimatedInputTokens = estimatePromptTokens(request.prompt);
-    if (estimatedInputTokens > maxSearchInputTokens) {
+      includeCounterReasonInPrompt: command.includeCounterReasonInPrompt,
+      maxSearchInputTokens,
+      baseRepomixConfig,
+    });
+
+    if (batches.batches.length === 0) {
       return {
         schemaVersion: "search.v1",
         mode: "search",
@@ -173,7 +175,7 @@ export async function runSearchCommand(command: SearchCommand, startedAt: number
         stats: {
           elapsedMs: Date.now() - startedAt,
           modelCalls: 0,
-          inputTokens: estimatedInputTokens,
+          inputTokens: batches.estimatedInputTokens,
           inputTokenCap: maxSearchInputTokens,
           skipped: true,
           skipReason: "input_too_large",
@@ -184,6 +186,8 @@ export async function runSearchCommand(command: SearchCommand, startedAt: number
           repomixFiles: pack.filePaths.length,
           repomixTokens: pack.totalTokens,
           repomixConfig: pack.config,
+          searchBatches: 0,
+          skippedTargets: batches.skippedTargets,
           profileId: profile?.id,
           targetsByPattern: countTargetsByPattern(searchContexts),
           targetsPreview: previewTargets(searchContexts),
@@ -192,43 +196,38 @@ export async function runSearchCommand(command: SearchCommand, startedAt: number
       };
     }
 
+    if (batches.wasSplit && !command.json) {
+      console.error(`Search input is large; queued ${batches.batches.length} smaller search batches.`);
+      if (batches.skippedTargets > 0) {
+        console.error(`Skipped ${batches.skippedTargets} oversized targets that could not fit alone.`);
+      }
+    }
+
     const modelPath = await firstRunModelBootstrap(command.model);
     const model = await loadLocalModel(modelPath, command.model, "scout");
-    const inputTokens = await countPromptTokens(model, request.prompt);
-    if (inputTokens > maxSearchInputTokens) {
-      return {
-        schemaVersion: "search.v1",
-        mode: "search",
-        source: command.source,
-        model: { id: command.model },
-        patterns: patternIds,
-        stats: {
-          elapsedMs: Date.now() - startedAt,
-          modelCalls: 0,
-          inputTokens,
-          inputTokenCap: maxSearchInputTokens,
-          skipped: true,
-          skipReason: "input_too_large",
-          filesChanged: changeSet.summary.fileCount,
-          entitiesScanned: changeSet.summary.total,
-          candidates: contexts.length,
-          searchTargets: searchContexts.length,
-          repomixFiles: pack.filePaths.length,
-          repomixTokens: pack.totalTokens,
-          repomixConfig: pack.config,
-          profileId: profile?.id,
-          targetsByPattern: countTargetsByPattern(searchContexts),
-          targetsPreview: previewTargets(searchContexts),
-        },
-        matches: [],
-      };
+    const matches = [];
+    let modelCalls = 0;
+    let inputTokens = 0;
+    let exactSkippedTargets = batches.skippedTargets;
+    for (const batch of batches.batches) {
+      const batchInputTokens = await countPromptTokens(model, batch.request.prompt);
+      inputTokens += batchInputTokens;
+      if (batchInputTokens > maxSearchInputTokens) {
+        exactSkippedTargets += batch.contexts.length;
+        if (!command.json) {
+          console.error(`Skipped ${batch.contexts.length} targets after exact token count exceeded the limit.`);
+        }
+        continue;
+      }
+      const { value } = await t.trace(
+        "search.model",
+        () => runSearch(model, batch.request),
+        { count: (v) => v.length },
+      );
+      modelCalls += 1;
+      matches.push(...value);
     }
-
-    const { value: matches } = await t.trace(
-      "search.model",
-      () => runSearch(model, request),
-      { count: (v) => v.length },
-    );
+    const uniqueMatches = dedupeMatches(matches);
 
     return {
       schemaVersion: "search.v1",
@@ -238,7 +237,7 @@ export async function runSearchCommand(command: SearchCommand, startedAt: number
       patterns: patternIds,
       stats: {
         elapsedMs: Date.now() - startedAt,
-        modelCalls: 1,
+        modelCalls,
         inputTokens,
         inputTokenCap: maxSearchInputTokens,
         filesChanged: changeSet.summary.fileCount,
@@ -248,17 +247,146 @@ export async function runSearchCommand(command: SearchCommand, startedAt: number
         repomixFiles: pack.filePaths.length,
         repomixTokens: pack.totalTokens,
         repomixConfig: pack.config,
+        searchBatches: batches.batches.length,
+        skippedTargets: exactSkippedTargets,
         profileId: profile?.id,
         targetsByPattern: countTargetsByPattern(searchContexts),
         targetsPreview: previewTargets(searchContexts),
       },
-      matches,
+      matches: uniqueMatches,
     };
   } finally {
     await changeSet.cleanup();
   }
 }
 
+function dedupeMatches<T extends { targetId: string; patternId: string; proof: string }>(matches: readonly T[]): readonly T[] {
+  const seen = new Set<string>();
+  return matches.filter((match) => {
+    const key = `${match.patternId}\n${match.proof.trim()}`;
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+}
+
+type SearchBatch = Readonly<{
+  contexts: readonly SemContext[];
+  pack: SemContextPack;
+  request: SearchRequest;
+  estimatedInputTokens: number;
+}>;
+
+async function buildSearchBatches(input: Readonly<{
+  command: SearchCommand;
+  changeSet: Parameters<typeof searchRequest>[0]["changeSet"];
+  contexts: readonly SemContext[];
+  initialPack: SemContextPack;
+  checks: readonly StupifyCheck[];
+  profile: SearchProfile | null;
+  includeCounterReasonInPrompt: boolean;
+  maxSearchInputTokens: number;
+  baseRepomixConfig: Parameters<typeof repomixContextPack>[3];
+}>): Promise<Readonly<{
+  batches: readonly SearchBatch[];
+  estimatedInputTokens: number;
+  skippedTargets: number;
+  wasSplit: boolean;
+}>> {
+  const first = makeSearchBatch(input, input.contexts, input.initialPack);
+  if (first.estimatedInputTokens <= input.maxSearchInputTokens) {
+    return {
+      batches: [first],
+      estimatedInputTokens: first.estimatedInputTokens,
+      skippedTargets: 0,
+      wasSplit: false,
+    };
+  }
+
+  const batches: SearchBatch[] = [];
+  let skippedTargets = 0;
+  let currentContexts: readonly SemContext[] = [];
+  let currentBatch: SearchBatch | null = null;
+
+  for (const context of input.contexts) {
+    const candidateContexts = [...currentContexts, context];
+    const candidateBatch = await makeSearchBatchWithPack(input, candidateContexts);
+    if (candidateBatch.estimatedInputTokens <= input.maxSearchInputTokens) {
+      currentContexts = candidateContexts;
+      currentBatch = candidateBatch;
+      continue;
+    }
+
+    if (currentBatch) {
+      batches.push(currentBatch);
+      currentContexts = [];
+      currentBatch = null;
+    }
+
+    const singleBatch = candidateContexts.length === 1
+      ? candidateBatch
+      : await makeSearchBatchWithPack(input, [context]);
+    if (singleBatch.estimatedInputTokens <= input.maxSearchInputTokens) {
+      currentContexts = [context];
+      currentBatch = singleBatch;
+    } else {
+      skippedTargets += 1;
+    }
+  }
+
+  if (currentBatch) batches.push(currentBatch);
+
+  return {
+    batches,
+    estimatedInputTokens: first.estimatedInputTokens,
+    skippedTargets,
+    wasSplit: true,
+  };
+}
+
+function makeSearchBatch(
+  input: Readonly<{
+    changeSet: Parameters<typeof searchRequest>[0]["changeSet"];
+    checks: readonly StupifyCheck[];
+    profile: SearchProfile | null;
+    includeCounterReasonInPrompt: boolean;
+  }>,
+  contexts: readonly SemContext[],
+  pack: SemContextPack,
+): SearchBatch {
+  const request = buildSearchRequest(
+    input.changeSet,
+    contexts,
+    pack,
+    input.checks,
+    input.profile,
+    input.includeCounterReasonInPrompt,
+  );
+  return {
+    contexts,
+    pack,
+    request,
+    estimatedInputTokens: estimatePromptTokens(request.prompt),
+  };
+}
+
+async function makeSearchBatchWithPack(
+  input: Readonly<{
+    command: SearchCommand;
+    changeSet: Parameters<typeof searchRequest>[0]["changeSet"];
+    checks: readonly StupifyCheck[];
+    profile: SearchProfile | null;
+    includeCounterReasonInPrompt: boolean;
+    baseRepomixConfig: Parameters<typeof repomixContextPack>[3];
+  }>,
+  contexts: readonly SemContext[],
+): Promise<SearchBatch> {
+  const pack = input.profile?.context === "sem"
+    ? emptyContextPack()
+    : await repomixContextPack(input.changeSet.contextCwd, contexts, input.changeSet.changes, input.baseRepomixConfig);
+  return makeSearchBatch(input, contexts, pack);
+}
+
 function buildSearchRequest(
   changeSet: Parameters<typeof searchRequest>[0]["changeSet"],
   contexts: Parameters<typeof searchRequest>[0]["contexts"],
@@ -302,7 +430,7 @@ function sourceLabel(command: SearchCommand): string {
 }
 
 function estimatePromptTokens(prompt: string): number {
-  return Math.ceil(prompt.length / 4);
+  return Math.ceil(prompt.length / 3);
 }
 
 function countTargetsByPattern(contexts: readonly SemContext[]): Record<string, number> {

package/src/types.ts

@@ -216,6 +216,8 @@ export type SearchRunJson = Readonly<{
     repomixTokens?: number;
     repomixConfig?: RepomixSearchConfig;
     searchTargets?: number;
+    searchBatches?: number;
+    skippedTargets?: number;
     profileId?: string;
     targetsByPattern?: Readonly<Record<string, number>>;
     targetsPreview?: readonly SearchTargetPreview[];