@llm4ts/flow 0.13.4 → 0.14.0

package/src/Survey.ts

@@ -66,25 +66,59 @@ export const closureFor = (
   return walk([program], new Set([program]), [])
 }
 
-const unitName = (path: string): string => {
+/**
+ * A source file's unit name: its basename without the extension. The graph
+ * keys nodes by it, and `resolveUnit` folds edge targets onto it.
+ */
+export const unitName = (path: string): string => {
   const base = path.split("/").at(-1) ?? path
   const dot = base.lastIndexOf(".")
   return dot < 0 ? base : base.slice(0, dot)
 }
 
+/**
+ * The unit a captured reference points at. COBOL rules capture the bare unit
+ * name (`CALL 'FEECALC'`), but web estates reference units by PATH —
+ * `<jsp:include page="header.jsp">`, `page="/WEB-INF/fragments/footer.jsp"`
+ * — so a raw capture would never equal a node name, every fragment would show
+ * zero incoming edges, and the inventory would flag the most-included files
+ * in the estate as retire candidates. Unknown references stay as captured:
+ * an edge to a unit the estate does not contain is itself a finding.
+ */
+export const resolveUnit = (reference: string, known: ReadonlySet<string>): string => {
+  if (known.has(reference)) {
+    return reference
+  }
+  const folded = unitName(reference)
+  return known.has(folded) ? folded : reference
+}
+
 const matches = (regex: string, contents: string): ReadonlyArray<string> => {
   const expression = new RegExp(regex, "g")
   return [...contents.matchAll(expression)].map((match) => match[1] ?? match[0])
 }
 
+export interface SurveyGraphOptions {
+  /** Regex over repo-relative paths to leave out even when `sources` matches. */
+  readonly exclude?: string
+}
+
 export const surveyGraph = Effect.fn("@llm4ts/flow/Survey.graph")(function* (
   workspace: WorkspaceShape,
   sources: string,
   units: ReadonlyArray<CoverageRule>,
-  edgeRules: ReadonlyArray<CoverageRule>
+  edgeRules: ReadonlyArray<CoverageRule>,
+  options: SurveyGraphOptions = {}
 ): Effect.fn.Return<SurveyGraph, WorkspaceError> {
-  const sourcePattern = new RegExp(sources)
-  const paths = (yield* workspace.discover()).filter((path) => sourcePattern.test(path)).sort()
+  // The source regex narrows discovery itself, so the workspace's result cap
+  // counts candidate units rather than every jar, image, and generated file
+  // sharing the tree with them.
+  const paths = [
+    ...(yield* workspace.discover("**/*", {
+      matching: new RegExp(sources),
+      ...(options.exclude === undefined ? {} : { excluding: new RegExp(options.exclude) })
+    }))
+  ].sort()
   const nodes: Array<SurveyNode> = []
   const contents = new Map<string, string>()
   for (const path of paths) {
@@ -101,6 +135,7 @@ export const surveyGraph = Effect.fn("@llm4ts/flow/Survey.graph")(function* (
       })
     )
   }
+  const known = new Set(nodes.map((node) => node.name))
   const edges: Array<SurveyEdge> = []
   for (const rule of edgeRules) {
     const filePattern = new RegExp(rule.files)
@@ -108,7 +143,7 @@ export const surveyGraph = Effect.fn("@llm4ts/flow/Survey.graph")(function* (
       for (const target of new Set(matches(rule.unit, contents.get(path) ?? ""))) {
         const edge = SurveyEdge.make({
           from: unitName(path),
-          to: target,
+          to: resolveUnit(target, known),
           kind: rule.name
         })
         if (
@@ -179,3 +214,100 @@ export const renderSurveyInventory = (graph: SurveyGraph): string => {
     ""
   ].join("\n")
 }
+
+/** The `graph.json` artifact: the graph as it is read back by later phases. */
+export const renderSurveyGraphJson = (graph: SurveyGraph): string =>
+  JSON.stringify(graph, undefined, 2)
+
+/**
+ * What a pack contributes to the survey's two reasoning prompts. The frame
+ * around it — the JSON contracts, the evidence rule, the wave discipline — is
+ * stack-neutral and lives here; everything that names a technology comes
+ * from the pack: the edge rules its graph was built from, and the optional
+ * `prompts/survey-refine.md` / `prompts/survey-triage.md` sidecars describing
+ * where THAT stack hides the links regexes miss and how to weigh its units.
+ */
+export interface SurveyPromptContext {
+  /** The pack's `## Survey:` edge rules — the graph's provenance, by name. */
+  readonly rules: ReadonlyArray<CoverageRule>
+  /** The pack's stack-specific guidance, or undefined for the neutral default. */
+  readonly guidance: string | undefined
+}
+
+const ruleNames = (context: SurveyPromptContext): string =>
+  context.rules.length === 0 ? "none" : context.rules.map((rule) => rule.name).join(", ")
+
+const defaultRefineGuidance = [
+  "Regexes miss links the source establishes indirectly: invocations whose target is held",
+  "in a variable or configuration entry, wiring declared in descriptors instead of code,",
+  "fragments pulled in by inclusion or templating, and units only a build or scheduler",
+  "step names."
+].join("\n")
+
+const defaultTriageGuidance = [
+  "Weigh each unit by what depends on it and what it depends on: shared units many others",
+  "reference are migrated early or wrapped; units nothing references are retire candidates",
+  "unless an entry point outside the graph (scheduler, external caller, deployment",
+  "descriptor) reaches them."
+].join("\n")
+
+export const surveyRefinePrompt = (graph: SurveyGraph, context: SurveyPromptContext): string =>
+  [
+    "You are refining the dependency graph of a legacy estate. The graph below was built",
+    "deterministically — one node per source file (named by its file name without the",
+    `extension), one edge per regex match of the pack's survey rules (${ruleNames(context)}).`,
+    context.guidance ?? defaultRefineGuidance,
+    'You have read-only access to the estate — read the sources (each node\'s "path" names its',
+    "file) and find the dependency edges the regexes missed. Prioritise the suspicious shapes:",
+    "units with fewer outgoing edges than the source suggests, units nothing references, units",
+    "with degree 0.",
+    "",
+    "Produce:",
+    '- "edges": ONLY links the graph does not already have, and ONLY between the units listed',
+    '  below (use the exact unit names). Each edge: "from" (the referencing unit), "to" (the',
+    '  referenced unit), "kind" (how the link is made — a short kebab-case label), and',
+    '  "evidence" (file, line, and the statement that establishes the link — no evidence, no',
+    "  edge). References to external systems, platform services, or third-party libraries are",
+    '  NOT edges; put them in "notes".',
+    '- "notes": references you could not resolve to a unit — indirect targets whose value you',
+    "  could not trace, external systems. Empty if none.",
+    "",
+    `Units: ${graph.nodes
+      .map((node) => node.name)
+      .sort()
+      .join(", ")}`,
+    "",
+    "Graph (JSON):",
+    renderSurveyGraphJson(graph)
+  ].join("\n")
+
+export const surveyTriagePrompt = (
+  graph: SurveyGraph,
+  inventory: string,
+  context: SurveyPromptContext
+): string =>
+  [
+    "You are triaging a legacy estate for modernization. Below are its inventory and",
+    `dependency graph (regex-derived from the source by the pack's survey rules — ${ruleNames(context)} —`,
+    "plus `llm-…` edges the graph-refine step grounded in the source with evidence — trust them).",
+    context.guidance ?? defaultTriageGuidance,
+    "",
+    "Produce:",
+    '- "triage": for EVERY unit in the inventory, a disposition:',
+    '  - "rewrite": actively used business logic or user-facing behaviour to modernize;',
+    '  - "retire": unreferenced/dead — candidate for decommissioning, with the evidence;',
+    '  - "wrap": keep on the legacy platform and front with an API (shared units other',
+    "    estates still call, or units out of this modernization's scope).",
+    "  Rationale in one sentence, grounded in the graph (degrees, callers, size).",
+    '- "waves": dependency-coherent migration slices for the REWRITE units: a wave\'s units',
+    "  should depend only on already-migrated or same-wave units where possible; leaves and",
+    "  low-fan-in units first; name each wave (wave-1, wave-2, …) and give the ordering rationale.",
+    '- "notes": anything the graph could not resolve — indirect references, cycles worth a',
+    "  human look. Empty if none.",
+    "",
+    "Inventory:",
+    inventory,
+    "",
+    "Graph (JSON):",
+    renderSurveyGraphJson(graph)
+  ].join("\n")

package/src/Workspace.ts

@@ -17,13 +17,34 @@ export interface WorkspaceLimits {
   readonly maxWriteBytes: number
   readonly maxResults: number
   readonly maxDepth: number
+  /**
+   * Directory names discovery never descends into, at any depth. Version
+   * control internals and dependency/build output are never estate sources,
+   * yet on a real repository they hold the overwhelming majority of files —
+   * walking them is slow and used to spend the result cap before the first
+   * source was seen. Absent means no pruning (the previous behaviour).
+   */
+  readonly excludeDirs?: ReadonlyArray<string>
 }
 
+/** The directories `defaultWorkspaceLimits` prunes from discovery. */
+export const defaultExcludedDirectories: ReadonlyArray<string> = Object.freeze([
+  ".git",
+  ".hg",
+  ".svn",
+  "node_modules",
+  "target",
+  "build",
+  "dist",
+  "out"
+])
+
 export const defaultWorkspaceLimits: WorkspaceLimits = Object.freeze({
   maxReadBytes: 1_048_576,
   maxWriteBytes: 1_048_576,
   maxResults: 1_000,
-  maxDepth: 32
+  maxDepth: 32,
+  excludeDirs: defaultExcludedDirectories
 })
 
 /**
@@ -32,30 +53,105 @@ export const defaultWorkspaceLimits: WorkspaceLimits = Object.freeze({
  * and generated exports, so the default 1 MiB read cap — sized for
  * spec-and-plan repositories — would fail an inventory on its first big
  * file. 8 MiB accommodates real estates while still refusing runaway blobs.
+ * The same reasoning sizes the discovery cap: an estate is by nature large,
+ * and 1 000 results — sized for spec-and-plan repositories — is fewer files
+ * than a mid-sized J2EE application ships in `src/` alone.
  */
 export const legacySourceWorkspaceLimits: WorkspaceLimits = Object.freeze({
   ...defaultWorkspaceLimits,
-  maxReadBytes: 8_388_608
+  maxReadBytes: 8_388_608,
+  maxResults: 20_000
 })
 
+const positiveInteger = (raw: string | undefined): number | undefined => {
+  const text = raw?.trim()
+  if (text === undefined || text.length === 0 || !/^\d+$/.test(text)) {
+    return undefined
+  }
+  const parsed = Number.parseInt(text, 10)
+  return Number.isSafeInteger(parsed) && parsed > 0 ? parsed : undefined
+}
+
+const directoryList = (raw: string | undefined): ReadonlyArray<string> | undefined => {
+  const text = raw?.trim()
+  if (text === undefined || text.length === 0) {
+    return undefined
+  }
+  const names = text
+    .split(",")
+    .map((name) => name.trim())
+    .filter((name) => name.length > 0 && !name.includes("/") && !name.includes("\\"))
+  return names.length === 0 ? undefined : names
+}
+
 /**
- * `defaults` with the per-file read cap overridden by `LLM4TS_MAX_READ_BYTES`
- * when it holds a positive integer; anything else leaves the defaults
- * untouched. The escape hatch for estates whose sources exceed even the
- * legacy-source cap.
+ * `defaults` with the caps overridden from the environment when the values
+ * are well-formed; anything else leaves that default untouched:
+ *
+ * - `LLM4TS_MAX_READ_BYTES` (positive integer): the per-file read cap, the
+ *   escape hatch for estates whose sources exceed even the legacy-source cap.
+ * - `LLM4TS_MAX_DISCOVER_RESULTS` (positive integer): the discovery result
+ *   cap, for estates larger than `legacySourceWorkspaceLimits` allows for.
+ * - `LLM4TS_EXCLUDE_DIRS` (comma-separated directory names): replaces the
+ *   pruned-directory list — `.git,node_modules,generated` — for estates
+ *   whose vendored or generated trees sit under names the default list
+ *   does not know.
  */
 export const workspaceLimitsFromEnv = (
   environment: Readonly<Record<string, string | undefined>>,
   defaults: WorkspaceLimits = defaultWorkspaceLimits
 ): WorkspaceLimits => {
-  const raw = environment.LLM4TS_MAX_READ_BYTES?.trim()
-  if (raw === undefined || raw.length === 0 || !/^\d+$/.test(raw)) {
+  const maxReadBytes = positiveInteger(environment.LLM4TS_MAX_READ_BYTES)
+  const maxResults = positiveInteger(environment.LLM4TS_MAX_DISCOVER_RESULTS)
+  const excludeDirs = directoryList(environment.LLM4TS_EXCLUDE_DIRS)
+  if (maxReadBytes === undefined && maxResults === undefined && excludeDirs === undefined) {
     return defaults
   }
-  const parsed = Number.parseInt(raw, 10)
-  return Number.isSafeInteger(parsed) && parsed > 0
-    ? { ...defaults, maxReadBytes: parsed }
-    : defaults
+  return {
+    ...defaults,
+    ...(maxReadBytes === undefined ? {} : { maxReadBytes }),
+    ...(maxResults === undefined ? {} : { maxResults }),
+    ...(excludeDirs === undefined ? {} : { excludeDirs })
+  }
+}
+
+/**
+ * Narrows a discovery beyond its glob: only paths `matching` the first regex
+ * and not `excluding` the second are returned AND counted against
+ * `maxResults`, so a cap sized for source units is not spent on the jars,
+ * images, and generated files that share the tree. Pass regexes without the
+ * `g` flag — a global regex carries `lastIndex` state across `test` calls.
+ */
+export interface DiscoverOptions {
+  readonly matching?: RegExp
+  readonly excluding?: RegExp
+}
+
+export const discoverAccepts = (path: string, options: DiscoverOptions): boolean =>
+  (options.matching?.test(path) ?? true) && !(options.excluding?.test(path) ?? false)
+
+/** Whether a repo-relative path lies under a pruned directory. */
+export const isExcludedPath = (path: string, limits: WorkspaceLimits): boolean => {
+  const excluded = limits.excludeDirs
+  if (excluded === undefined || excluded.length === 0) {
+    return false
+  }
+  const directories = path.split("/").slice(0, -1)
+  return directories.some((segment) => excluded.includes(segment))
+}
+
+/**
+ * The advice a flow gives when discovery overflows `limits.maxResults`: the
+ * three knobs that narrow or raise it, in the order a user should try them.
+ */
+export const discoveryOverflowAdvice = (limits: WorkspaceLimits): string => {
+  const pruned = limits.excludeDirs ?? []
+  return (
+    `discovery stopped at ${limits.maxResults} matching files; ` +
+    "narrow the pack's `sources:` regex or add an `exclude:` regex, prune more directories " +
+    `with LLM4TS_EXCLUDE_DIRS=<names> (pruned now: ${pruned.length === 0 ? "none" : pruned.join(",")}), ` +
+    "or raise the cap with LLM4TS_MAX_DISCOVER_RESULTS=<count>"
+  )
 }
 
 export interface WorkspaceShape {
@@ -64,7 +160,10 @@ export interface WorkspaceShape {
   readonly read: (path: string) => Effect.Effect<string, WorkspaceError>
   readonly write: (path: string, contents: string) => Effect.Effect<void, WorkspaceError>
   readonly append: (path: string, contents: string) => Effect.Effect<void, WorkspaceError>
-  readonly discover: (pattern?: string) => Effect.Effect<ReadonlyArray<string>, WorkspaceError>
+  readonly discover: (
+    pattern?: string,
+    options?: DiscoverOptions
+  ) => Effect.Effect<ReadonlyArray<string>, WorkspaceError>
   readonly search: (
     query: string,
     pattern?: string
@@ -210,13 +309,19 @@ export const makeMemoryWorkspace = (
         )
       }
 
-      const discover = (pattern = "**/*"): Effect.Effect<ReadonlyArray<string>, WorkspaceError> =>
+      const discover = (
+        pattern = "**/*",
+        options: DiscoverOptions = {}
+      ): Effect.Effect<ReadonlyArray<string>, WorkspaceError> =>
         Effect.gen(function* () {
           const normalizedPattern = pattern.replaceAll("\\", "/")
           const matcher = normalizedPattern === "**/*" ? /.*/ : globRegex(normalizedPattern)
           const files = yield* Ref.get(state)
           const results: Array<string> = []
           for (const path of Object.keys(files).sort(comparePaths)) {
+            if (isExcludedPath(path, limits)) {
+              continue
+            }
             const depth = path.split("/").length - 1
             if (depth > limits.maxDepth) {
               return yield* WorkspaceLimitError.make({
@@ -225,7 +330,7 @@ export const makeMemoryWorkspace = (
                 actual: depth
               })
             }
-            if (matcher.test(path)) {
+            if (matcher.test(path) && discoverAccepts(path, options)) {
               results.push(path)
               if (results.length > limits.maxResults) {
                 return yield* WorkspaceLimitError.make({