@neurocode-ai/core 1.18.8

package/src/ripgrep.ts

@@ -0,0 +1,281 @@
+export * as Ripgrep from "./ripgrep"
+
+import { Context, Effect, Fiber, Layer, Schema, Stream } from "effect"
+import { ChildProcess } from "effect/unstable/process"
+import { Entry, Match } from "@neurocode-ai/schema/filesystem"
+import { makeGlobalNode } from "./effect/app-node"
+import { AppProcess, collectStream, waitForAbort } from "./process"
+import { NonNegativeInt, PositiveInt, RelativePath } from "./schema"
+import { RipgrepBinary } from "./ripgrep/binary"
+
+/**
+ * Small core-owned ripgrep execution adapter. It deliberately exposes raw
+ * process-oriented rows, not model text or permission behavior. Search maps
+ * these rows into filesystem results; leaf tools own
+ * presentation and permission prompts.
+ */
+
+const ERROR_BYTES = 8 * 1024
+const MAX_RECORD_BYTES = 64 * 1024
+const MAX_SUBMATCHES = 100
+
+const RawMatch = Schema.Struct({
+  type: Schema.Literal("match"),
+  data: Schema.Struct({
+    path: Schema.Struct({ text: Schema.String }),
+    lines: Schema.Struct({ text: Schema.String }),
+    line_number: PositiveInt,
+    absolute_offset: NonNegativeInt,
+    submatches: Schema.Array(
+      Schema.Struct({
+        match: Schema.Struct({ text: Schema.String }),
+        start: NonNegativeInt,
+        end: NonNegativeInt,
+      }),
+    ),
+  }),
+})
+
+type RawMatchData = (typeof RawMatch.Type)["data"]
+
+export class Error extends Schema.TaggedErrorClass<Error>()("Ripgrep.Error", {
+  message: Schema.String,
+  cause: Schema.optional(Schema.Defect()),
+}) {}
+
+export class InvalidPatternError extends Schema.TaggedErrorClass<InvalidPatternError>()("Ripgrep.InvalidPatternError", {
+  pattern: Schema.String,
+  message: Schema.String,
+}) {}
+
+export interface FindInput {
+  readonly cwd: string
+  readonly pattern: string
+  readonly limit: number
+  readonly hidden?: boolean
+  readonly follow?: boolean
+  readonly signal?: AbortSignal
+  readonly onEntry?: (entry: Entry) => Effect.Effect<void>
+}
+
+export interface GlobInput {
+  readonly cwd: string
+  readonly pattern: string
+  readonly limit: number
+  readonly hidden?: boolean
+  readonly follow?: boolean
+  readonly signal?: AbortSignal
+}
+
+export interface GrepInput {
+  readonly cwd: string
+  readonly pattern: string
+  readonly file?: string
+  readonly include?: string
+  readonly limit: number
+  readonly signal?: AbortSignal
+}
+
+export interface Interface {
+  readonly find: (input: FindInput) => Effect.Effect<readonly Entry[], Error>
+  readonly glob: (input: GlobInput) => Effect.Effect<readonly Entry[], Error>
+  readonly grep: (input: GrepInput) => Effect.Effect<readonly Match[], Error | InvalidPatternError>
+}
+
+export class Service extends Context.Service<Service, Interface>()("@neurocode/v2/Ripgrep") {}
+
+const failure = (message: string, cause?: unknown) => new Error({ message, cause })
+
+const isInvalidPattern = (stderr: string) =>
+  stderr.includes("regex parse error") || stderr.includes("error parsing regex")
+
+const layer = Layer.effect(
+  Service,
+  Effect.gen(function* () {
+    const process = yield* AppProcess.Service
+    const binary = yield* RipgrepBinary.Service
+
+    const run = <A>(input: {
+      readonly cwd: string
+      readonly args: string[]
+      readonly limit: number
+      readonly signal?: AbortSignal
+      readonly parse: (line: string) => Effect.Effect<A | undefined, Error>
+      readonly pattern?: string
+      readonly onItem?: (item: A) => Effect.Effect<void>
+    }) => {
+      const program = Effect.scoped(
+        Effect.gen(function* () {
+          const handle = yield* process.spawn(
+            ChildProcess.make(yield* binary.filepath, input.args, { cwd: input.cwd, extendEnv: true, stdin: "ignore" }),
+          )
+          const stderrFiber = yield* collectStream(handle.stderr, ERROR_BYTES).pipe(
+            Effect.map((output) => output.buffer.toString("utf8")),
+            Effect.forkScoped,
+          )
+          let observed = 0
+          const rows = yield* Stream.decodeText(handle.stdout).pipe(
+            Stream.splitLines,
+            Stream.filter((line) => line.length > 0),
+            Stream.mapEffect(input.parse),
+            Stream.filter((row): row is A => row !== undefined),
+            Stream.tap((row) => {
+              if (!input.onItem || observed++ >= input.limit) return Effect.void
+              return input.onItem(row)
+            }),
+            Stream.take(input.limit + 1),
+            Stream.runCollect,
+            Effect.map((chunk) => [...chunk]),
+          )
+          const truncated = rows.length > input.limit
+          if (truncated) return { items: rows.slice(0, input.limit), truncated, partial: false }
+
+          const code = yield* handle.exitCode
+          const stderr = yield* Fiber.join(stderrFiber)
+          if (input.pattern && code === 2 && isInvalidPattern(stderr)) {
+            return yield* new InvalidPatternError({ pattern: input.pattern, message: stderr.trim() })
+          }
+          if (code !== 0 && code !== 1 && code !== 2) {
+            return yield* failure(stderr.trim() || `ripgrep failed with code ${code}`)
+          }
+          return { items: code === 1 ? [] : rows, truncated: false, partial: code === 2 }
+        }),
+      )
+      const abortable = input.signal ? program.pipe(Effect.raceFirst(waitForAbort(input.signal))) : program
+      return abortable.pipe(
+        Effect.mapError((cause) =>
+          cause instanceof Error || cause instanceof InvalidPatternError
+            ? cause
+            : failure("ripgrep execution failed", cause),
+        ),
+      )
+    }
+
+    return Service.of({
+      glob: (input) =>
+        run<string>({
+          cwd: input.cwd,
+          limit: input.limit,
+          signal: input.signal,
+          args: [
+            "--no-config",
+            "--files",
+            ...(input.hidden ? ["--hidden"] : []),
+            ...(input.follow ? ["--follow"] : []),
+            `--glob=${input.pattern}`,
+            "--glob=!**/.git/**",
+            ".",
+          ],
+          parse: (line) =>
+            Effect.succeed(
+              line
+                .replace(/^(?:\.[\\/])+/u, "")
+                .replace(/^[\\/]+/u, "")
+                .replaceAll("\\", "/"),
+            ),
+        }).pipe(
+          Effect.map((result) =>
+            result.items.map((relative) =>
+              Entry.make({
+                path: RelativePath.make(relative),
+                type: "file",
+              }),
+            ),
+          ),
+          Effect.catchTag("Ripgrep.InvalidPatternError", (cause) => Effect.fail(failure(cause.message, cause))),
+        ),
+      find: (input) =>
+        run<Entry>({
+          cwd: input.cwd,
+          limit: input.limit,
+          signal: input.signal,
+          args: [
+            "--no-config",
+            "--files",
+            ...(input.hidden ? ["--hidden"] : []),
+            ...(input.follow ? ["--follow"] : []),
+            ...(input.pattern === "*" ? [] : [`--glob=${input.pattern}`]),
+            "--glob=!**/.git/**",
+            ".",
+          ],
+          parse: (line) => {
+            const relative = line
+              .replace(/^(?:\.[\\/])+/u, "")
+              .replace(/^[\\/]+/u, "")
+              .replaceAll("\\", "/")
+            return Effect.succeed(
+              Entry.make({
+                path: RelativePath.make(relative),
+                type: "file",
+              }),
+            )
+          },
+          onItem: input.onEntry,
+        }).pipe(
+          Effect.map((result) => result.items),
+          Effect.catchTag("Ripgrep.InvalidPatternError", (cause) => Effect.fail(failure(cause.message, cause))),
+        ),
+      grep: (input) =>
+        run<RawMatchData>({
+          ...input,
+          args: [
+            "--no-config",
+            "--json",
+            "--hidden",
+            "--no-messages",
+            ...(input.include ? [`--glob=${input.include}`] : []),
+            "--glob=!**/.git/**",
+            "--",
+            input.pattern,
+            input.file ?? ".",
+          ],
+          parse: (line) =>
+            (Buffer.byteLength(line, "utf8") > MAX_RECORD_BYTES
+              ? Effect.fail(failure(`Ripgrep JSON record exceeded ${MAX_RECORD_BYTES} bytes`))
+              : Effect.try({
+                  try: () => JSON.parse(line) as unknown,
+                  catch: (cause) => failure("Invalid ripgrep JSON output", cause),
+                })
+            ).pipe(
+              Effect.flatMap((json) => {
+                if (!json || typeof json !== "object" || !("type" in json) || json.type !== "match")
+                  return Effect.succeed(undefined)
+                return Schema.decodeUnknownEffect(RawMatch)(json).pipe(
+                  Effect.map((match) => ({
+                    ...match.data,
+                    path: { text: match.data.path.text.replace(/^\.[\\/]/, "") },
+                    submatches: match.data.submatches.slice(0, MAX_SUBMATCHES),
+                  })),
+                  Effect.mapError((cause) => failure("Invalid ripgrep match output", cause)),
+                )
+              }),
+            ),
+        }).pipe(
+          Effect.map((result) =>
+            result.items.map((match) => {
+              const relative = match.path.text
+                .replace(/^(?:\.[\\/])+/u, "")
+                .replace(/^[\\/]+/u, "")
+                .replaceAll("\\", "/")
+              return Match.make({
+                entry: Entry.make({
+                  path: RelativePath.make(relative),
+                  type: "file",
+                }),
+                line: match.line_number,
+                offset: match.absolute_offset,
+                text: match.lines.text.length > 2_000 ? match.lines.text.slice(0, 2_000) + "..." : match.lines.text,
+                submatches: match.submatches.map((submatch) => ({
+                  text: submatch.match.text,
+                  start: submatch.start,
+                  end: submatch.end,
+                })),
+              })
+            }),
+          ),
+        ),
+    })
+  }),
+)
+
+export const node = makeGlobalNode({ service: Service, layer: layer, deps: [RipgrepBinary.node, AppProcess.node] })

package/src/schema.ts

@@ -0,0 +1,79 @@
+import { Schema } from "effect"
+import {
+  AbsolutePath,
+  DateTimeUtcFromMillis,
+  NonNegativeInt,
+  optional,
+  PositiveInt,
+  RelativePath,
+  statics,
+} from "@neurocode-ai/schema/schema"
+
+export { AbsolutePath, DateTimeUtcFromMillis, NonNegativeInt, optional, PositiveInt, RelativePath, statics }
+
+/**
+ * Strip `readonly` from a nested type. Stand-in for `effect`'s `Types.DeepMutable`
+ * until `effect:core/x228my` ("Types.DeepMutable widens unknown to `{}`") lands.
+ *
+ * The upstream version falls through `unknown` into `{ -readonly [K in keyof T]: ... }`
+ * where `keyof unknown = never`, so `unknown` collapses to `{}`. This local
+ * version gates the object branch on `extends object` (which `unknown` does
+ * not) so `unknown` passes through untouched.
+ *
+ * Primitive bailout matches upstream — without it, branded strings like
+ * `string & Brand<"SessionID">` fall into the object branch and get their
+ * prototype methods walked.
+ *
+ * Tuple branch preserves readonly tuples (e.g. `ConfigPlugin.Spec`'s
+ * `readonly [string, Options]`); the general array branch would otherwise
+ * widen them to unbounded arrays.
+ */
+// eslint-disable-next-line @typescript-eslint/ban-types
+export type DeepMutable<T> = T extends string | number | boolean | bigint | symbol | Function
+  ? T
+  : T extends readonly [unknown, ...unknown[]]
+    ? { -readonly [K in keyof T]: DeepMutable<T[K]> }
+    : T extends readonly (infer U)[]
+      ? DeepMutable<U>[]
+      : T extends object
+        ? { -readonly [K in keyof T]: DeepMutable<T[K]> }
+        : T
+
+/**
+ * Nominal wrapper for scalar types. The class itself is a valid schema —
+ * pass it directly to `Schema.decode`, `Schema.decodeEffect`, etc.
+ *
+ * Overrides `~type.make` on the derived `Schema.Opaque` so `Schema.Schema.Type`
+ * of a field using this newtype resolves to `Self` rather than the underlying
+ * branded phantom. Without that override, passing a class instance to code
+ * typed against `Schema.Schema.Type<FieldSchema>` would require a cast even
+ * though the values are structurally equivalent at runtime.
+ *
+ * @example
+ *   class QuestionID extends Newtype<QuestionID>()("QuestionID", Schema.String) {
+ *     static make(id: string): QuestionID {
+ *       return this.make(id)
+ *     }
+ *   }
+ *
+ *   Schema.decodeEffect(QuestionID)(input)
+ */
+export function Newtype<Self>() {
+  return <const Tag extends string, S extends Schema.Top>(tag: Tag, schema: S) => {
+    abstract class Base {
+      declare readonly _newtype: Tag
+
+      static make(value: Schema.Schema.Type<S>): Self {
+        return value as unknown as Self
+      }
+    }
+
+    Object.setPrototypeOf(Base, schema)
+
+    return Base as unknown as (abstract new (_: never) => { readonly _newtype: Tag }) & {
+      readonly make: (value: Schema.Schema.Type<S>) => Self
+    } & Omit<Schema.Opaque<Self, S, {}>, "make" | "~type.make"> & {
+        readonly "~type.make": Self
+      }
+  }
+}

package/src/session/compaction.ts

@@ -0,0 +1,241 @@
+export * as SessionCompaction from "./compaction"
+
+import { LLM, LLMError, LLMEvent, Message, type LLMRequest, type Model } from "@neurocode-ai/llm"
+import { DateTime, Effect, Stream } from "effect"
+import type { Config } from "../config"
+import type { EventV2 } from "../event"
+import { SessionEvent } from "./event"
+import { SessionMessage } from "./message"
+import { SessionSchema } from "./schema"
+import { Token } from "../util/token"
+
+const DEFAULT_BUFFER = 20_000
+const DEFAULT_KEEP_TOKENS = 8_000
+const TOOL_OUTPUT_MAX_CHARS = 2_000
+const SUMMARY_OUTPUT_TOKENS = 4_096
+const SUMMARY_TEMPLATE = `Output exactly the Markdown structure shown inside <template> and keep the section order unchanged. Do not include the <template> tags in your response.
+<template>
+## Objective
+- [one or two brief sentences describing what the user is trying to accomplish]
+
+## Important Details
+- [constraints/preferences, decisions and why, important facts/assumptions, exact context needed to continue, or "(none)"]
+
+## Work State
+### Completed
+- [finished work, verified facts, or changes made; otherwise "(none)"]
+
+### Active
+- [current work, partial changes, or investigation state; otherwise "(none)"]
+
+### Blocked
+- [blockers, failing commands, or unknowns; otherwise "(none)"]
+
+## Next Move
+1. [immediate concrete action, or "(none)"]
+2. [next action if known, or "(none)"]
+
+## Relevant Files
+- [file or directory path: why it matters, or "(none)"]
+</template>
+
+Rules:
+- Keep every section, even when empty.
+- Use terse bullets, not prose paragraphs.
+- Preserve exact file paths, symbols, commands, error strings, URLs, and identifiers when known.
+- Do not mention the summary process or that context was compacted.`
+
+type Entry = {
+  readonly seq: number
+  readonly message: SessionMessage.Message
+}
+
+type Settings = {
+  readonly auto: boolean
+  readonly buffer: number
+  readonly tokens: number
+}
+
+type Dependencies = {
+  readonly events: EventV2.Interface
+  readonly llm: {
+    readonly stream: (request: LLMRequest) => Stream.Stream<LLMEvent, LLMError>
+  }
+  readonly config: readonly Config.Entry[]
+}
+
+type Input = {
+  readonly sessionID: SessionSchema.ID
+  readonly entries: readonly Entry[]
+  readonly model: Model
+  readonly request: LLMRequest
+}
+
+const estimate = (value: unknown) => Token.estimate(JSON.stringify(value))
+
+const truncate = (value: string) =>
+  value.length <= TOOL_OUTPUT_MAX_CHARS ? value : `${value.slice(0, TOOL_OUTPUT_MAX_CHARS)}\n[truncated]`
+
+export const serializeToolContent = (content: SessionMessage.ToolStateCompleted["content"]) =>
+  content
+    .map((item) =>
+      item.type === "text" ? item.text : `[Attached ${item.mime}${item.name === undefined ? "" : `: ${item.name}`}]`,
+    )
+    .join("\n")
+
+const serialize = (message: SessionMessage.Message) => {
+  if (message.type === "user") {
+    const files = message.files?.map((file) => `[Attached ${file.mime}: ${file.name ?? file.uri}]`) ?? []
+    return [`[User]: ${message.text}`, ...files].join("\n")
+  }
+  if (message.type === "assistant") {
+    return message.content
+      .flatMap((part) => {
+        if (part.type === "text") return [`[Assistant]: ${part.text}`]
+        if (part.type === "reasoning") return part.text ? [`[Assistant reasoning]: ${part.text}`] : []
+        const input = typeof part.state.input === "string" ? part.state.input : JSON.stringify(part.state.input)
+        if (part.state.status === "completed")
+          return [
+            `[Assistant tool call]: ${part.name}(${input})`,
+            `[Tool result]: ${truncate(serializeToolContent(part.state.content))}`,
+          ]
+        if (part.state.status === "error")
+          return [`[Assistant tool call]: ${part.name}(${input})`, `[Tool error]: ${part.state.error.message}`]
+        return [`[Assistant tool call]: ${part.name}(${input})`]
+      })
+      .join("\n")
+  }
+  if (message.type === "system") return `[System update]: ${message.text}`
+  if (message.type === "synthetic") return `[Synthetic context]: ${message.text}`
+  if (message.type === "shell") return `[Shell]: ${message.command}\n${truncate(message.output)}`
+  return ""
+}
+
+const settings = (documents: readonly Config.Entry[]) => {
+  const configured = documents
+    .filter((entry): entry is Config.Document => entry.type === "document")
+    .flatMap((entry) => (entry.info.compaction ? [entry.info.compaction] : []))
+  return configured.reduce<Settings>(
+    (result, current) => ({
+      auto: current.auto ?? result.auto,
+      buffer: current.buffer ?? result.buffer,
+      tokens: current.keep?.tokens ?? result.tokens,
+    }),
+    { auto: true, buffer: DEFAULT_BUFFER, tokens: DEFAULT_KEEP_TOKENS },
+  )
+}
+
+const select = (
+  entries: readonly Entry[],
+  tokens: number,
+): { readonly head: string; readonly recent: string } | undefined => {
+  const conversation = entries
+    .filter((entry) => entry.message.type !== "compaction")
+    .map((entry) => serialize(entry.message))
+    .filter(Boolean)
+  if (conversation.length === 0) return
+  let total = 0
+  let split = conversation.length
+  let splitPrefix = ""
+  let splitSuffix = ""
+  for (let index = conversation.length - 1; index >= 0; index--) {
+    const next = total + Token.estimate(conversation[index])
+    if (next > tokens) {
+      const remaining = Math.max(0, tokens - total) * 4
+      if (remaining > 0) {
+        splitPrefix = conversation[index].slice(0, -remaining)
+        splitSuffix = conversation[index].slice(-remaining)
+        split = index + 1
+      }
+      break
+    }
+    total = next
+    split = index
+  }
+  return {
+    head: [...conversation.slice(0, split), splitPrefix].filter(Boolean).join("\n\n"),
+    recent: [splitSuffix, ...conversation.slice(split)].filter(Boolean).join("\n\n"),
+  }
+}
+
+export const buildPrompt = (input: { readonly previousSummary?: string; readonly context: readonly string[] }) =>
+  [
+    input.previousSummary
+      ? `Update the anchored summary below using the conversation history above.\nPreserve still-true details, remove stale details, and merge in the new facts.\n<previous-summary>\n${input.previousSummary}\n</previous-summary>`
+      : "Create a new anchored summary from the conversation history.",
+    SUMMARY_TEMPLATE,
+    ...input.context,
+  ].join("\n\n")
+
+export const make = (dependencies: Dependencies) => {
+  const config = settings(dependencies.config)
+  const compactAfterOverflow = Effect.fn("SessionCompaction.compactAfterOverflow")(function* (input: Input) {
+    const context = input.model.route.defaults.limits?.context
+    if (context === undefined || context <= 0) return false
+    const output = input.request.generation?.maxTokens ?? input.model.route.defaults.limits?.output ?? 0
+    const selected = select(input.entries, config.tokens)
+    const previousSummary = input.entries.find((entry) => entry.message.type === "compaction")?.message
+    if (!selected || (selected.head.length === 0 && previousSummary?.type !== "compaction")) return false
+    const summaryPrompt = buildPrompt({
+      previousSummary: previousSummary?.type === "compaction" ? previousSummary.summary : undefined,
+      context: [previousSummary?.type === "compaction" ? previousSummary.recent : "", selected.head].filter(Boolean),
+    })
+    const summaryOutput = Math.min(output || SUMMARY_OUTPUT_TOKENS, SUMMARY_OUTPUT_TOKENS)
+    if (Token.estimate(summaryPrompt) > context - summaryOutput) return false
+    const messageID = SessionMessage.ID.create()
+    yield* dependencies.events.publish(SessionEvent.Compaction.Started, {
+      sessionID: input.sessionID,
+      messageID,
+      timestamp: yield* DateTime.now,
+      reason: "auto",
+    })
+
+    const chunks: string[] = []
+    let failed = false
+    const summarized = yield* dependencies.llm
+      .stream(
+        LLM.request({
+          model: input.model,
+          messages: [Message.user(summaryPrompt)],
+          tools: [],
+          generation: { maxTokens: summaryOutput },
+        }),
+      )
+      .pipe(
+        Stream.runForEach((event) => {
+          if (LLMEvent.is.providerError(event)) failed = true
+          if (LLMEvent.is.textDelta(event)) chunks.push(event.text)
+          return Effect.void
+        }),
+        Effect.as(true),
+        Effect.catchTag("LLM.Error", () => Effect.succeed(false)),
+      )
+    const summary = chunks.join("")
+    if (!summarized || failed || !summary.trim()) return false
+    yield* dependencies.events.publish(SessionEvent.Compaction.Ended, {
+      sessionID: input.sessionID,
+      messageID,
+      timestamp: yield* DateTime.now,
+      reason: "auto",
+      text: summary,
+      recent: selected.recent,
+    })
+    return true
+  })
+  const compactIfNeeded = Effect.fn("SessionCompaction.compactIfNeeded")(function* (input: Input) {
+    if (!config.auto) return false
+    const context = input.model.route.defaults.limits?.context
+    if (context === undefined || context <= 0) return false
+    const output = input.request.generation?.maxTokens ?? input.model.route.defaults.limits?.output ?? 0
+    if (
+      estimate({ system: input.request.system, messages: input.request.messages, tools: input.request.tools }) <=
+      context - Math.max(output, config.buffer)
+    )
+      return false
+    return yield* compactAfterOverflow(input)
+  })
+  return {
+    compactIfNeeded,
+    compactAfterOverflow,
+  }
+}