@neurocode-ai/core 1.18.8

package/src/shell.ts

@@ -0,0 +1,226 @@
+export * as Shell from "./shell"
+
+import path from "path"
+import { spawn, type ChildProcess } from "child_process"
+import { readFile } from "fs/promises"
+import { statSync } from "fs"
+import { setTimeout as sleep } from "node:timers/promises"
+import { Flag } from "./flag/flag"
+import { FSUtil } from "./fs-util"
+import { which } from "./util/which"
+
+const SIGKILL_TIMEOUT_MS = 200
+const META: Record<string, { deny?: boolean; login?: boolean; posix?: boolean; ps?: boolean }> = {
+  bash: { login: true, posix: true },
+  dash: { login: true, posix: true },
+  fish: { deny: true, login: true },
+  ksh: { login: true, posix: true },
+  nu: { deny: true },
+  powershell: { ps: true },
+  pwsh: { ps: true },
+  sh: { login: true, posix: true },
+  zsh: { login: true, posix: true },
+}
+
+export type Item = {
+  path: string
+  name: string
+  acceptable: boolean
+}
+
+export async function killTree(proc: ChildProcess, opts?: { exited?: () => boolean }): Promise<void> {
+  const pid = proc.pid
+  if (!pid || opts?.exited?.()) return
+
+  if (process.platform === "win32") {
+    await new Promise<void>((resolve) => {
+      const killer = spawn("taskkill", ["/pid", String(pid), "/f", "/t"], {
+        stdio: "ignore",
+        windowsHide: true,
+      })
+      killer.once("exit", () => resolve())
+      killer.once("error", () => resolve())
+    })
+    return
+  }
+
+  try {
+    process.kill(-pid, "SIGTERM")
+    await sleep(SIGKILL_TIMEOUT_MS)
+    if (!opts?.exited?.()) {
+      process.kill(-pid, "SIGKILL")
+    }
+  } catch {
+    proc.kill("SIGTERM")
+    await sleep(SIGKILL_TIMEOUT_MS)
+    if (!opts?.exited?.()) {
+      proc.kill("SIGKILL")
+    }
+  }
+}
+
+function stat(file: string) {
+  return statSync(file, { throwIfNoEntry: false }) ?? undefined
+}
+
+function full(file: string) {
+  if (process.platform !== "win32") return file
+  const shell = FSUtil.windowsPath(file)
+  if (path.win32.dirname(shell) !== ".") {
+    if (shell.startsWith("/") && name(shell) === "bash") return gitbash() || shell
+    return shell
+  }
+  if (name(shell) === "bash") return gitbash() || which(shell) || shell
+  return which(shell) || shell
+}
+
+function meta(file: string) {
+  return META[name(file)]
+}
+
+function ok(file: string) {
+  return meta(file)?.deny !== true
+}
+
+function rooted(file: string) {
+  return path.isAbsolute(FSUtil.windowsPath(file))
+}
+
+function resolve(file: string) {
+  const shell = full(file)
+  if (rooted(shell)) {
+    if (stat(shell)?.isFile()) return shell
+    return
+  }
+  return which(shell) ?? undefined
+}
+
+function win() {
+  return Array.from(
+    new Set(
+      [which("pwsh"), which("powershell"), gitbash(), process.env.COMSPEC || "cmd.exe"]
+        .filter((item): item is string => Boolean(item))
+        .map(full),
+    ),
+  )
+}
+
+async function unix() {
+  const text = await readFile("/etc/shells", "utf8").catch(() => "")
+  if (text) return Array.from(new Set(text.split("\n").filter((line) => line.trim() && !line.startsWith("#"))))
+  return ["/bin/bash", "/bin/zsh", "/bin/sh"]
+}
+
+function select(file: string | undefined, opts?: { acceptable?: boolean }) {
+  if (file && (!opts?.acceptable || ok(file))) {
+    const shell = resolve(file)
+    if (shell) return shell
+  }
+  if (process.platform === "win32") return win()[0]
+  return fallback()
+}
+
+export function gitbash() {
+  if (process.platform !== "win32") return
+  if (Flag.NEUROCODE_GIT_BASH_PATH) return Flag.NEUROCODE_GIT_BASH_PATH
+  const git = which("git")
+  if (!git) return
+  const file = path.join(git, "..", "..", "bin", "bash.exe")
+  if (stat(file)?.size) return file
+}
+
+function fallback() {
+  if (process.platform === "darwin") return "/bin/zsh"
+  const bash = which("bash")
+  if (bash) return bash
+  return "/bin/sh"
+}
+
+export function name(file: string) {
+  if (process.platform === "win32") return path.win32.parse(FSUtil.windowsPath(file)).name.toLowerCase()
+  return path.basename(file).toLowerCase()
+}
+
+export function login(file: string) {
+  return meta(file)?.login === true
+}
+
+export function posix(file: string) {
+  return meta(file)?.posix === true
+}
+
+export function ps(file: string) {
+  return meta(file)?.ps === true
+}
+
+function info(file: string): Item {
+  const item = full(file)
+  const n = name(item)
+  return {
+    path: item,
+    name: resolve(n) ? n : item,
+    acceptable: ok(item),
+  }
+}
+
+export function args(file: string, command: string, cwd: string) {
+  const n = name(file)
+  if (n === "nu" || n === "fish") return ["-c", command]
+  if (n === "zsh") {
+    return [
+      "-l",
+      "-c",
+      `
+        [[ -f ~/.zshenv ]] && source ~/.zshenv >/dev/null 2>&1 || true
+        [[ -f "\${ZDOTDIR:-$HOME}/.zshrc" ]] && source "\${ZDOTDIR:-$HOME}/.zshrc" >/dev/null 2>&1 || true
+        cd -- "$1"
+        eval ${JSON.stringify(command)}
+      `,
+      "neurocode",
+      cwd,
+    ]
+  }
+  if (n === "bash") {
+    return [
+      "-l",
+      "-c",
+      `
+        shopt -s expand_aliases
+        [[ -f ~/.bashrc ]] && source ~/.bashrc >/dev/null 2>&1 || true
+        cd -- "$1"
+        eval ${JSON.stringify(command)}
+      `,
+      "neurocode",
+      cwd,
+    ]
+  }
+  if (n === "cmd") return ["/c", command]
+  if (ps(file)) return ["-NoProfile", "-Command", command]
+  return ["-c", command]
+}
+
+let defaultPreferred: string | undefined
+let defaultAcceptable: string | undefined
+
+export function preferred(configShell?: string) {
+  if (configShell) return select(configShell)
+  defaultPreferred ??= select(process.env.SHELL)
+  return defaultPreferred
+}
+preferred.reset = () => {
+  defaultPreferred = undefined
+}
+
+export function acceptable(configShell?: string) {
+  if (configShell) return select(configShell, { acceptable: true })
+  defaultAcceptable ??= select(process.env.SHELL, { acceptable: true })
+  return defaultAcceptable
+}
+acceptable.reset = () => {
+  defaultAcceptable = undefined
+}
+
+export async function list(): Promise<Item[]> {
+  const shells = process.platform === "win32" ? win() : await unix()
+  return shells.filter((s) => resolve(s)).map(info)
+}

package/src/skill/discovery.ts

@@ -0,0 +1,213 @@
+export * as SkillDiscovery from "./discovery"
+
+import path from "path"
+import { Context, Effect, Layer, Schedule, Schema } from "effect"
+import { FetchHttpClient, HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
+import { FSUtil } from "../fs-util"
+import { Global } from "../global"
+import { makeGlobalNode } from "../effect/app-node"
+import { httpClient } from "../effect/app-node-platform"
+import { AbsolutePath } from "../schema"
+
+const skillConcurrency = 4
+const fileConcurrency = 8
+
+function isSafeSegment(value: string) {
+  return (
+    value.length > 0 &&
+    value !== "." &&
+    value !== ".." &&
+    !value.includes("/") &&
+    !value.includes("\\") &&
+    !value.includes("\0")
+  )
+}
+
+function isSafeRelativePath(value: string) {
+  const segments = value.split("/")
+  return (
+    value.length > 0 &&
+    !value.includes("\\") &&
+    !value.includes("\0") &&
+    !value.includes("?") &&
+    !value.includes("#") &&
+    !URL.canParse(value) &&
+    !path.posix.isAbsolute(value) &&
+    !path.win32.isAbsolute(value) &&
+    segments.every((segment) => {
+      try {
+        const decoded = decodeURIComponent(segment)
+        return (
+          decoded.length > 0 &&
+          decoded !== "." &&
+          decoded !== ".." &&
+          !decoded.includes("/") &&
+          !decoded.includes("\\") &&
+          !decoded.includes("\0")
+        )
+      } catch {
+        return false
+      }
+    })
+  )
+}
+
+class IndexSkill extends Schema.Class<IndexSkill>("SkillDiscovery.IndexSkill")({
+  name: Schema.String,
+  version: Schema.optional(Schema.String),
+  files: Schema.Array(Schema.String),
+}) {}
+
+class Index extends Schema.Class<Index>("SkillDiscovery.Index")({
+  skills: Schema.Array(IndexSkill),
+}) {}
+
+export interface Interface {
+  readonly pull: (url: string) => Effect.Effect<AbsolutePath[]>
+}
+
+export class Service extends Context.Service<Service, Interface>()("@neurocode/v2/SkillDiscovery") {}
+
+const layer = Layer.effect(
+  Service,
+  Effect.gen(function* () {
+    const fs = yield* FSUtil.Service
+    const global = yield* Global.Service
+    const http = (yield* HttpClient.HttpClient).pipe(
+      HttpClient.retryTransient({
+        retryOn: "errors-and-responses",
+        times: 2,
+        schedule: Schedule.exponential(200).pipe(Schedule.jittered),
+      }),
+      HttpClient.filterStatusOk,
+    )
+
+    const download = Effect.fn("SkillDiscovery.download")(function* (url: string, destination: string) {
+      if (yield* fs.exists(destination).pipe(Effect.orDie)) return true
+      return yield* HttpClientRequest.get(url).pipe(
+        http.execute,
+        Effect.flatMap((response) => response.arrayBuffer),
+        Effect.flatMap((body) => fs.writeWithDirs(destination, new Uint8Array(body))),
+        Effect.as(true),
+        Effect.catch((error) =>
+          Effect.logError("failed to download skill file", { url, error }).pipe(Effect.as(false)),
+        ),
+      )
+    })
+
+    return Service.of({
+      pull: Effect.fn("SkillDiscovery.pull")(function* (url) {
+        const base = url.endsWith("/") ? url : `${url}/`
+        const source = new URL(base)
+        const index = new URL("index.json", source).href
+        const data = yield* HttpClientRequest.get(index).pipe(
+          HttpClientRequest.acceptJson,
+          http.execute,
+          Effect.flatMap(HttpClientResponse.schemaBodyJson(Index)),
+          Effect.catch((error) =>
+            Effect.logError("failed to fetch skill index", { url: index, error }).pipe(Effect.as(undefined)),
+          ),
+        )
+        if (!data) return []
+
+        const sourceRoot = path.resolve(global.cache, "skills", Bun.hash(base).toString(16))
+        return yield* Effect.forEach(
+          data.skills.flatMap((skill) => {
+            if (!isSafeSegment(skill.name)) {
+              return []
+            }
+            if (!skill.files.includes("SKILL.md") && !skill.files.includes(`${skill.name}.md`)) {
+              return []
+            }
+
+            const root = path.resolve(sourceRoot, skill.name)
+            if (!FSUtil.contains(sourceRoot, root) || root === sourceRoot) {
+              return []
+            }
+
+            const skillUrl = new URL(`${encodeURIComponent(skill.name)}/`, source)
+            const versionFile = path.join(root, ".neurocode-version")
+            const files = skill.files.map((file) => {
+              if (!isSafeRelativePath(file)) return undefined
+              let resource: URL
+              try {
+                resource = new URL(file, skillUrl)
+              } catch {
+                return undefined
+              }
+              if (resource.origin !== source.origin) return undefined
+
+              const destination = path.resolve(root, file)
+              if (!FSUtil.contains(root, destination) || destination === root) return undefined
+              return {
+                url: resource.href,
+                destination,
+                file,
+              }
+            })
+            if (files.some((file) => file === undefined)) {
+              return []
+            }
+            return [{ skill, root, versionFile, files: files as { url: string; destination: string; file: string }[] }]
+          }),
+          ({ skill, root, versionFile, files }) =>
+            Effect.gen(function* () {
+              const version = skill.version
+              const current =
+                version === undefined
+                  ? undefined
+                  : yield* fs.readFileStringSafe(versionFile).pipe(Effect.catch(() => Effect.succeed(undefined)))
+              if (version === undefined || current === version) {
+                yield* Effect.forEach(files, (file) => download(file.url, file.destination), {
+                  concurrency: fileConcurrency,
+                  discard: true,
+                })
+              } else {
+                const token = crypto.randomUUID()
+                const staging = `${root}.tmp-${token}`
+                const backup = `${root}.old-${token}`
+                yield* Effect.gen(function* () {
+                  const downloaded = yield* Effect.forEach(
+                    files,
+                    (file) => download(file.url, path.resolve(staging, file.file)),
+                    { concurrency: fileConcurrency },
+                  )
+                  if (!downloaded.every(Boolean)) return
+                  const exists =
+                    (yield* fs.exists(path.join(staging, "SKILL.md")).pipe(Effect.orDie)) ||
+                    (yield* fs.exists(path.join(staging, `${skill.name}.md`)).pipe(Effect.orDie))
+                  if (!exists) return
+                  yield* fs.writeFileString(path.join(staging, ".neurocode-version"), version)
+                  yield* Effect.uninterruptible(
+                    Effect.gen(function* () {
+                      const cached = yield* fs.exists(root).pipe(Effect.orDie)
+                      if (cached) yield* fs.rename(root, backup)
+                      yield* fs.rename(staging, root).pipe(
+                        Effect.catch((error) =>
+                          Effect.gen(function* () {
+                            if (cached) yield* fs.rename(backup, root).pipe(Effect.ignore)
+                            return yield* Effect.fail(error)
+                          }),
+                        ),
+                      )
+                      if (cached) yield* fs.remove(backup, { recursive: true, force: true }).pipe(Effect.ignore)
+                    }),
+                  )
+                }).pipe(
+                  Effect.catch((error) => Effect.logError("failed to refresh skill", { skill: skill.name, error })),
+                  Effect.ensuring(fs.remove(staging, { recursive: true, force: true }).pipe(Effect.ignore)),
+                )
+              }
+              const exists =
+                (yield* fs.exists(path.join(root, "SKILL.md")).pipe(Effect.orDie)) ||
+                (yield* fs.exists(path.join(root, `${skill.name}.md`)).pipe(Effect.orDie))
+              return exists ? [AbsolutePath.make(root)] : []
+            }),
+          { concurrency: skillConcurrency },
+        ).pipe(Effect.map((directories) => directories.flat()))
+      }),
+    })
+  }),
+)
+
+export const node = makeGlobalNode({ service: Service, layer, deps: [httpClient, FSUtil.node, Global.node] })

package/src/skill/guidance.ts

@@ -0,0 +1,76 @@
+export * as SkillGuidance from "./guidance"
+
+import { makeLocationNode } from "../effect/app-node"
+import { Context, Effect, Layer, Schema } from "effect"
+import { AgentV2 } from "../agent"
+import { PermissionV2 } from "../permission"
+import { SkillV2 } from "../skill"
+import { SystemContext } from "../system-context/index"
+
+const Summary = Schema.Struct({
+  name: Schema.String,
+  description: Schema.String,
+})
+type Summary = typeof Summary.Type
+
+const render = (skills: ReadonlyArray<Summary>) =>
+  [
+    "Skills provide specialized instructions and workflows for specific tasks.",
+    "Use the skill tool to load a skill when a task matches its description.",
+    ...(skills.length === 0
+      ? ["No skills are currently available."]
+      : [
+          "<available_skills>",
+          ...skills.flatMap((skill) => [
+            "  <skill>",
+            `    <name>${skill.name}</name>`,
+            `    <description>${skill.description}</description>`,
+            "  </skill>",
+          ]),
+          "</available_skills>",
+        ]),
+  ].join("\n")
+
+export interface Interface {
+  readonly load: (agent: AgentV2.Selection) => Effect.Effect<SystemContext.SystemContext>
+}
+
+export class Service extends Context.Service<Service, Interface>()("@neurocode/v2/SkillGuidance") {}
+
+const layer = Layer.effect(
+  Service,
+  Effect.gen(function* () {
+    const skills = yield* SkillV2.Service
+
+    return Service.of({
+      load: Effect.fn("SkillGuidance.load")(function* (selection) {
+        const agent = selection.info
+        if (!agent) return SystemContext.empty
+        const permitted = SkillV2.available(yield* skills.list(), agent)
+        if (permitted.length === 0 && PermissionV2.evaluate("skill", "*", agent.permissions).effect === "deny")
+          return SystemContext.empty
+        const available = permitted
+          .flatMap((skill) =>
+            skill.description === undefined ? [] : [{ name: skill.name, description: skill.description }],
+          )
+          .toSorted((a, b) => a.name.localeCompare(b.name))
+        return SystemContext.make({
+          key: SystemContext.Key.make("core/skill-guidance"),
+          codec: Schema.toCodecJson(Schema.Array(Summary)),
+          load: Effect.succeed(available),
+          baseline: render,
+          update: (_previous, current) =>
+            [
+              "The available skills have changed. This list supersedes the previous available skills list.",
+              render(current),
+            ].join("\n"),
+          removed: () => "Skill guidance is no longer available. Do not use any previously listed skill.",
+        })
+      }),
+    })
+  }),
+)
+
+export const locationLayer = layer
+
+export const node = makeLocationNode({ service: Service, layer, deps: [SkillV2.node] })

package/src/skill.ts

@@ -0,0 +1,132 @@
+export * as SkillV2 from "./skill"
+
+import { makeLocationNode } from "./effect/app-node"
+import path from "path"
+import { Context, Effect, Layer, Schema, Types } from "effect"
+import { Skill } from "@neurocode-ai/schema/skill"
+import { AgentV2 } from "./agent"
+import { ConfigMarkdown } from "./config/markdown"
+import { FSUtil } from "./fs-util"
+import { PermissionV2 } from "./permission"
+import { AbsolutePath } from "./schema"
+import { SkillDiscovery } from "./skill/discovery"
+import { State } from "./state"
+
+export const DirectorySource = Skill.DirectorySource
+export type DirectorySource = Skill.DirectorySource
+
+export const UrlSource = Skill.UrlSource
+export type UrlSource = Skill.UrlSource
+
+export const EmbeddedSource = Skill.EmbeddedSource
+export type EmbeddedSource = Skill.EmbeddedSource
+
+export const Source = Skill.Source
+export type Source = typeof Source.Type
+
+export const Info = Skill.Info
+export type Info = Skill.Info
+
+export const available = (skills: ReadonlyArray<Info>, agent: AgentV2.Info) =>
+  skills.filter((skill) => PermissionV2.evaluate("skill", skill.name, agent.permissions).effect !== "deny")
+
+const Frontmatter = Schema.Struct({
+  name: Schema.String.pipe(Schema.optional),
+  description: Schema.String.pipe(Schema.optional),
+  slash: Schema.Boolean.pipe(Schema.optional),
+})
+const decodeFrontmatter = Schema.decodeUnknownOption(Frontmatter)
+
+export type Data = {
+  sources: Types.DeepMutable<Source>[]
+}
+
+export type Draft = {
+  source: (source: Source) => void
+  list: () => readonly Source[]
+}
+
+export interface Interface extends State.Transformable<Draft> {
+  readonly sources: () => Effect.Effect<Source[]>
+  readonly list: () => Effect.Effect<Info[]>
+}
+
+export class Service extends Context.Service<Service, Interface>()("@neurocode/v2/Skill") {}
+
+const layer = Layer.effect(
+  Service,
+  Effect.gen(function* () {
+    const discovery = yield* SkillDiscovery.Service
+    const fs = yield* FSUtil.Service
+
+    const state = State.create<Data, Draft>({
+      initial: () => ({ sources: [] }),
+      draft: (draft) => ({
+        source: (source) => {
+          if (draft.sources.some((item) => Source.equals(item, source))) return
+          draft.sources.push(source as Types.DeepMutable<Source>)
+        },
+        list: () => draft.sources as Source[],
+      }),
+    })
+
+    const load = Effect.fn("SkillV2.load")(function* (source: Source) {
+      const skills: Info[] = []
+      if (source.type === "embedded") return [source.skill]
+      const directories = source.type === "directory" ? [source.path] : yield* discovery.pull(source.url)
+      for (const directory of directories) {
+        const files = yield* fs
+          .glob("{*.md,**/SKILL.md}", { cwd: directory, absolute: true, include: "file", symlink: true, dot: true })
+          .pipe(Effect.catch(() => Effect.succeed([] as string[])))
+        for (const filepath of files.toSorted()) {
+          const content = yield* fs.readFileStringSafe(filepath).pipe(Effect.catch(() => Effect.succeed(undefined)))
+          if (!content) continue
+          const markdown = ConfigMarkdown.parseOption(content)
+          if (!markdown) continue
+          const frontmatter = decodeFrontmatter(markdown.data).valueOrUndefined
+          if (!frontmatter) continue
+          const name =
+            frontmatter.name !== undefined
+              ? frontmatter.name
+              : path.dirname(filepath) === directory
+                ? path.basename(filepath, ".md")
+                : undefined
+          if (!name) continue
+          skills.push({
+            name,
+            description: frontmatter.description,
+            slash: frontmatter.slash,
+            location: AbsolutePath.make(filepath),
+            content: markdown.content,
+          })
+        }
+      }
+      return skills
+    })
+
+    // QUESTION(Dax): Should local skill sources invalidate on filesystem watch
+    // events, following the reload policy chosen for other context sources?
+    const cache = new Map<string, Info[]>()
+    const list = Effect.fn("SkillV2.list")(function* () {
+      const skills = new Map<string, Info>()
+      for (const source of state.get().sources) {
+        const key = Source.key(source)
+        const loaded = cache.get(key) ?? (yield* load(source))
+        cache.set(key, loaded)
+        for (const skill of loaded) skills.set(skill.name, skill)
+      }
+      return Array.from(skills.values())
+    })
+
+    return Service.of({
+      transform: state.transform,
+      reload: state.reload,
+      sources: Effect.fn("SkillV2.sources")(function* () {
+        return state.get().sources
+      }),
+      list,
+    })
+  }),
+)
+
+export const node = makeLocationNode({ service: Service, layer, deps: [SkillDiscovery.node, FSUtil.node] })