@llm4ts/flow 0.13.5 → 0.14.0

package/src/Workspace.ts

@@ -17,13 +17,34 @@ export interface WorkspaceLimits {
   readonly maxWriteBytes: number
   readonly maxResults: number
   readonly maxDepth: number
+  /**
+   * Directory names discovery never descends into, at any depth. Version
+   * control internals and dependency/build output are never estate sources,
+   * yet on a real repository they hold the overwhelming majority of files —
+   * walking them is slow and used to spend the result cap before the first
+   * source was seen. Absent means no pruning (the previous behaviour).
+   */
+  readonly excludeDirs?: ReadonlyArray<string>
 }
 
+/** The directories `defaultWorkspaceLimits` prunes from discovery. */
+export const defaultExcludedDirectories: ReadonlyArray<string> = Object.freeze([
+  ".git",
+  ".hg",
+  ".svn",
+  "node_modules",
+  "target",
+  "build",
+  "dist",
+  "out"
+])
+
 export const defaultWorkspaceLimits: WorkspaceLimits = Object.freeze({
   maxReadBytes: 1_048_576,
   maxWriteBytes: 1_048_576,
   maxResults: 1_000,
-  maxDepth: 32
+  maxDepth: 32,
+  excludeDirs: defaultExcludedDirectories
 })
 
 /**
@@ -32,30 +53,105 @@ export const defaultWorkspaceLimits: WorkspaceLimits = Object.freeze({
  * and generated exports, so the default 1 MiB read cap — sized for
  * spec-and-plan repositories — would fail an inventory on its first big
  * file. 8 MiB accommodates real estates while still refusing runaway blobs.
+ * The same reasoning sizes the discovery cap: an estate is by nature large,
+ * and 1 000 results — sized for spec-and-plan repositories — is fewer files
+ * than a mid-sized J2EE application ships in `src/` alone.
  */
 export const legacySourceWorkspaceLimits: WorkspaceLimits = Object.freeze({
   ...defaultWorkspaceLimits,
-  maxReadBytes: 8_388_608
+  maxReadBytes: 8_388_608,
+  maxResults: 20_000
 })
 
+const positiveInteger = (raw: string | undefined): number | undefined => {
+  const text = raw?.trim()
+  if (text === undefined || text.length === 0 || !/^\d+$/.test(text)) {
+    return undefined
+  }
+  const parsed = Number.parseInt(text, 10)
+  return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : undefined
+}
+
+const directoryList = (raw: string | undefined): ReadonlyArray<string> | undefined => {
+  const text = raw?.trim()
+  if (text === undefined || text.length === 0) {
+    return undefined
+  }
+  const names = text
+    .split(",")
+    .map((name) => name.trim())
+    .filter((name) => name.length > 0 && !name.includes("/") && !name.includes("\\"))
+  return names.length === 0 ? undefined : names
+}
+
 /**
- * `defaults` with the per-file read cap overridden by `LLM4TS_MAX_READ_BYTES`
- * when it holds a positive integer; anything else leaves the defaults
- * untouched. The escape hatch for estates whose sources exceed even the
- * legacy-source cap.
+ * `defaults` with the caps overridden from the environment when the values
+ * are well-formed; anything else leaves that default untouched:
+ *
+ * - `LLM4TS_MAX_READ_BYTES` (positive integer): the per-file read cap, the
+ *   escape hatch for estates whose sources exceed even the legacy-source cap.
+ * - `LLM4TS_MAX_DISCOVER_RESULTS` (positive integer): the discovery result
+ *   cap, for estates larger than `legacySourceWorkspaceLimits` allows for.
+ * - `LLM4TS_EXCLUDE_DIRS` (comma-separated directory names): replaces the
+ *   pruned-directory list — `.git,node_modules,generated` — for estates
+ *   whose vendored or generated trees sit under names the default list
+ *   does not know.
  */
 export const workspaceLimitsFromEnv = (
   environment: Readonly<Record<string, string | undefined>>,
   defaults: WorkspaceLimits = defaultWorkspaceLimits
 ): WorkspaceLimits => {
-  const raw = environment.LLM4TS_MAX_READ_BYTES?.trim()
-  if (raw === undefined || raw.length === 0 || !/^\d+$/.test(raw)) {
+  const maxReadBytes = positiveInteger(environment.LLM4TS_MAX_READ_BYTES)
+  const maxResults = positiveInteger(environment.LLM4TS_MAX_DISCOVER_RESULTS)
+  const excludeDirs = directoryList(environment.LLM4TS_EXCLUDE_DIRS)
+  if (maxReadBytes === undefined && maxResults === undefined && excludeDirs === undefined) {
     return defaults
   }
-  const parsed = Number.parseInt(raw, 10)
-  return Number.isSafeInteger(parsed) && parsed > 0
-    ? { ...defaults, maxReadBytes: parsed }
-    : defaults
+  return {
+    ...defaults,
+    ...(maxReadBytes === undefined ? {} : { maxReadBytes }),
+    ...(maxResults === undefined ? {} : { maxResults }),
+    ...(excludeDirs === undefined ? {} : { excludeDirs })
+  }
+}
+
+/**
+ * Narrows a discovery beyond its glob: only paths `matching` the first regex
+ * and not `excluding` the second are returned AND counted against
+ * `maxResults`, so a cap sized for source units is not spent on the jars,
+ * images, and generated files that share the tree. Pass regexes without the
+ * `g` flag — a global regex carries `lastIndex` state across `test` calls.
+ */
+export interface DiscoverOptions {
+  readonly matching?: RegExp
+  readonly excluding?: RegExp
+}
+
+export const discoverAccepts = (path: string, options: DiscoverOptions): boolean =>
+  (options.matching?.test(path) ?? true) && !(options.excluding?.test(path) ?? false)
+
+/** Whether a repo-relative path lies under a pruned directory. */
+export const isExcludedPath = (path: string, limits: WorkspaceLimits): boolean => {
+  const excluded = limits.excludeDirs
+  if (excluded === undefined || excluded.length === 0) {
+    return false
+  }
+  const directories = path.split("/").slice(0, -1)
+  return directories.some((segment) => excluded.includes(segment))
+}
+
+/**
+ * The advice a flow gives when discovery overflows `limits.maxResults`: the
+ * three knobs that narrow or raise it, in the order a user should try them.
+ */
+export const discoveryOverflowAdvice = (limits: WorkspaceLimits): string => {
+  const pruned = limits.excludeDirs ?? []
+  return (
+    `discovery stopped at ${limits.maxResults} matching files; ` +
+    "narrow the pack's `sources:` regex or add an `exclude:` regex, prune more directories " +
+    `with LLM4TS_EXCLUDE_DIRS=<names> (pruned now: ${pruned.length === 0 ? "none" : pruned.join(",")}), ` +
+    "or raise the cap with LLM4TS_MAX_DISCOVER_RESULTS=<count>"
+  )
 }
 
 export interface WorkspaceShape {
@@ -64,7 +160,10 @@ export interface WorkspaceShape {
   readonly read: (path: string) => Effect.Effect<string, WorkspaceError>
   readonly write: (path: string, contents: string) => Effect.Effect<void, WorkspaceError>
   readonly append: (path: string, contents: string) => Effect.Effect<void, WorkspaceError>
-  readonly discover: (pattern?: string) => Effect.Effect<ReadonlyArray<string>, WorkspaceError>
+  readonly discover: (
+    pattern?: string,
+    options?: DiscoverOptions
+  ) => Effect.Effect<ReadonlyArray<string>, WorkspaceError>
   readonly search: (
     query: string,
     pattern?: string
@@ -210,13 +309,19 @@ export const makeMemoryWorkspace = (
         )
       }
 
-      const discover = (pattern = "**/*"): Effect.Effect<ReadonlyArray<string>, WorkspaceError> =>
+      const discover = (
+        pattern = "**/*",
+        options: DiscoverOptions = {}
+      ): Effect.Effect<ReadonlyArray<string>, WorkspaceError> =>
         Effect.gen(function* () {
           const normalizedPattern = pattern.replaceAll("\\", "/")
           const matcher = normalizedPattern === "**/*" ? /.*/ : globRegex(normalizedPattern)
           const files = yield* Ref.get(state)
           const results: Array<string> = []
           for (const path of Object.keys(files).sort(comparePaths)) {
+            if (isExcludedPath(path, limits)) {
+              continue
+            }
             const depth = path.split("/").length - 1
             if (depth > limits.maxDepth) {
               return yield* WorkspaceLimitError.make({
@@ -225,7 +330,7 @@ export const makeMemoryWorkspace = (
                 actual: depth
               })
             }
-            if (matcher.test(path)) {
+            if (matcher.test(path) && discoverAccepts(path, options)) {
               results.push(path)
               if (results.length > limits.maxResults) {
                 return yield* WorkspaceLimitError.make({