agentikit 0.0.7 → 0.0.8

package/src/markdown.ts

@@ -0,0 +1,106 @@
+import { parseFrontmatter } from "./frontmatter"
+
+// ── Types ───────────────────────────────────────────────────────────────────
+
+export interface TocHeading {
+  level: number
+  text: string
+  line: number
+}
+
+export interface KnowledgeToc {
+  headings: TocHeading[]
+  totalLines: number
+}
+
+// ── Parsing ─────────────────────────────────────────────────────────────────
+
+export function parseMarkdownToc(content: string): KnowledgeToc {
+  const lines = content.split(/\r?\n/)
+  const headings: TocHeading[] = []
+
+  const parsed = parseFrontmatter(content)
+  const start = parsed.frontmatter ? parsed.bodyStartLine - 1 : 0
+
+  for (let i = start; i < lines.length; i++) {
+    const match = lines[i].match(/^(#{1,6})\s+(.+)$/)
+    if (match) {
+      headings.push({
+        level: match[1].length,
+        text: match[2].replace(/\s+#+\s*$/, "").trim(),
+        line: i + 1,
+      })
+    }
+  }
+
+  return { headings, totalLines: lines.length }
+}
+
+// ── Extraction ──────────────────────────────────────────────────────────────
+
+export function extractSection(
+  content: string,
+  heading: string,
+): { content: string; startLine: number; endLine: number } | null {
+  const lines = content.split(/\r?\n/)
+  const target = heading.toLowerCase()
+
+  let startIdx = -1
+  let startLevel = 0
+
+  for (let i = 0; i < lines.length; i++) {
+    const match = lines[i].match(/^(#{1,6})\s+(.+)$/)
+    if (!match) continue
+    const text = match[2].replace(/\s+#+\s*$/, "").trim()
+    if (text.toLowerCase() === target && startIdx === -1) {
+      startIdx = i
+      startLevel = match[1].length
+    } else if (startIdx !== -1 && match[1].length <= startLevel) {
+      return {
+        content: lines.slice(startIdx, i).join("\n"),
+        startLine: startIdx + 1,
+        endLine: i,
+      }
+    }
+  }
+
+  if (startIdx === -1) return null
+
+  return {
+    content: lines.slice(startIdx).join("\n"),
+    startLine: startIdx + 1,
+    endLine: lines.length,
+  }
+}
+
+export function extractLineRange(content: string, start: number, end: number): string {
+  const lines = content.split(/\r?\n/)
+  if (end < start) return ""
+  const s = Math.max(1, Math.min(start, lines.length))
+  const e = Math.min(end, lines.length)
+  return lines.slice(s - 1, e).join("\n")
+}
+
+export function extractFrontmatterOnly(content: string): string | null {
+  const parsed = parseFrontmatter(content)
+  return parsed.frontmatter
+}
+
+// ── Formatting ──────────────────────────────────────────────────────────────
+
+export function formatToc(toc: KnowledgeToc): string {
+  if (toc.headings.length === 0) {
+    return `(no headings found — ${toc.totalLines} lines total)`
+  }
+
+  const lineWidth = String(toc.totalLines).length
+  const parts = toc.headings.map((h) => {
+    const lineNum = `L${String(h.line).padStart(lineWidth)}`
+    const indent = "  ".repeat(h.level - 1)
+    const prefix = "#".repeat(h.level)
+    return `${lineNum}  ${indent}${prefix} ${h.text}`
+  })
+
+  parts.push(`\n${toc.totalLines} lines total`)
+  return parts.join("\n")
+}

package/src/metadata.ts

@@ -1,6 +1,9 @@
 import fs from "node:fs"
 import path from "node:path"
-import type { AgentikitAssetType } from "./stash"
+import { type AgentikitAssetType, isAssetType } from "./common"
+import { SCRIPT_EXTENSIONS, isRelevantAssetFile, deriveCanonicalAssetName } from "./asset-spec"
+import { parseFrontmatter, toStringOrUndefined } from "./frontmatter"
+import { parseMarkdownToc, type TocHeading } from "./markdown"
 
 // ── Schema ──────────────────────────────────────────────────────────────────
 
@@ -16,9 +19,15 @@ export interface StashEntry {
   description?: string
   tags?: string[]
   examples?: string[]
+  intents?: string[]
   intent?: StashIntent
   entry?: string
   generated?: boolean
+  quality?: "generated" | "curated"
+  confidence?: number
+  source?: "package" | "frontmatter" | "comments" | "filename" | "manual" | "llm"
+  aliases?: string[]
+  toc?: TocHeading[]
 }
 
 export interface StashFile {
@@ -59,7 +68,7 @@ export function validateStashEntry(entry: unknown): StashEntry | null {
   if (typeof entry !== "object" || entry === null) return null
   const e = entry as Record<string, unknown>
   if (typeof e.name !== "string" || !e.name) return null
-  if (typeof e.type !== "string" || !isValidType(e.type)) return null
+  if (typeof e.type !== "string" || !isAssetType(e.type)) return null
 
   const result: StashEntry = {
     name: e.name,
@@ -68,6 +77,10 @@ export function validateStashEntry(entry: unknown): StashEntry | null {
   if (typeof e.description === "string" && e.description) result.description = e.description
   if (Array.isArray(e.tags)) result.tags = e.tags.filter((t): t is string => typeof t === "string")
   if (Array.isArray(e.examples)) result.examples = e.examples.filter((x): x is string => typeof x === "string")
+  if (Array.isArray(e.intents)) {
+    const filtered = e.intents.filter((s): s is string => typeof s === "string" && s.trim().length > 0)
+    if (filtered.length > 0) result.intents = filtered
+  }
   if (typeof e.intent === "object" && e.intent !== null) {
     const intent = e.intent as Record<string, unknown>
     result.intent = {}
@@ -77,67 +90,120 @@ export function validateStashEntry(entry: unknown): StashEntry | null {
   }
   if (typeof e.entry === "string" && e.entry) result.entry = e.entry
   if (e.generated === true) result.generated = true
+  if (e.quality === "generated" || e.quality === "curated") result.quality = e.quality
+  if (typeof e.confidence === "number" && Number.isFinite(e.confidence)) result.confidence = Math.max(0, Math.min(1, e.confidence))
+  if (typeof e.source === "string" && ["package", "frontmatter", "comments", "filename", "manual", "llm"].includes(e.source)) {
+    result.source = e.source as StashEntry["source"]
+  }
+  if (Array.isArray(e.aliases)) {
+    const filtered = e.aliases.filter((a): a is string => typeof a === "string" && a.trim().length > 0)
+    if (filtered.length > 0) result.aliases = normalizeTerms(filtered)
+  }
+  if (Array.isArray(e.toc)) {
+    const validated = e.toc.filter(
+      (h: unknown): h is TocHeading => {
+        if (typeof h !== "object" || h === null) return false
+        const rec = h as Record<string, unknown>
+        return typeof rec.level === "number"
+          && typeof rec.text === "string"
+          && typeof rec.line === "number"
+      },
+    )
+    if (validated.length > 0) result.toc = validated
+  }
 
   return result
 }
 
-function isValidType(type: string): boolean {
-  return type === "tool" || type === "skill" || type === "command" || type === "agent"
-}
-
 // ── Metadata Generation ─────────────────────────────────────────────────────
 
-const SCRIPT_EXTENSIONS = new Set([".sh", ".ts", ".js", ".ps1", ".cmd", ".bat"])
-
 export function generateMetadata(
   dirPath: string,
   assetType: AgentikitAssetType,
   files: string[],
+  typeRoot = dirPath,
 ): StashFile {
   const entries: StashEntry[] = []
+  const pkgMeta = extractPackageMetadata(dirPath)
 
   for (const file of files) {
     const ext = path.extname(file).toLowerCase()
     const baseName = path.basename(file, ext)
+    const fileName = path.basename(file)
 
     // Skip non-relevant files
-    if (assetType === "tool" && !SCRIPT_EXTENSIONS.has(ext)) continue
-    if ((assetType === "command" || assetType === "agent") && ext !== ".md") continue
-    if (assetType === "skill" && path.basename(file) !== "SKILL.md") continue
+    if (!isRelevantAssetFile(assetType, fileName)) continue
+
+    const canonicalName = assetType === "skill"
+      ? deriveCanonicalAssetName(assetType, typeRoot, file) ?? baseName
+      : baseName
 
     const entry: StashEntry = {
-      name: baseName,
+      name: canonicalName,
       type: assetType,
       generated: true,
+      quality: "generated",
+      confidence: 0.55,
+      source: "filename",
     }
 
-    // Priority 3: package.json metadata
-    const pkgMeta = extractPackageMetadata(dirPath)
+    // Priority 1: package.json metadata
     if (pkgMeta) {
-      if (pkgMeta.description && !entry.description) entry.description = pkgMeta.description
-      if (pkgMeta.keywords && pkgMeta.keywords.length > 0) entry.tags = pkgMeta.keywords
+      if (pkgMeta.description && !entry.description) {
+        entry.description = pkgMeta.description
+        entry.source = "package"
+        entry.confidence = 0.8
+      }
+      if (pkgMeta.keywords && pkgMeta.keywords.length > 0) entry.tags = normalizeTerms(pkgMeta.keywords)
     }
 
-    // Priority 2: Frontmatter (for .md files)
+    // Priority 2: Frontmatter (for .md files — overrides package.json description)
     if (ext === ".md") {
       const fm = extractFrontmatterDescription(file)
-      if (fm) entry.description = fm
+      if (fm) {
+        entry.description = fm
+        entry.source = "frontmatter"
+        entry.confidence = 0.9
+      }
+    }
+
+    // Knowledge entries: generate TOC from headings
+    if (assetType === "knowledge") {
+      try {
+        const mdContent = fs.readFileSync(file, "utf8")
+        const toc = parseMarkdownToc(mdContent)
+        if (toc.headings.length > 0) entry.toc = toc.headings
+      } catch {
+        // Non-fatal: skip TOC if file can't be read
+      }
     }
 
-    // Priority 4: Code comments (for script files)
+    // Priority 3: Code comments (for script files)
     if (SCRIPT_EXTENSIONS.has(ext) && ext !== ".md") {
       const commentDesc = extractDescriptionFromComments(file)
-      if (commentDesc && !entry.description) entry.description = commentDesc
+      if (commentDesc && !entry.description) {
+        entry.description = commentDesc
+        entry.source = "comments"
+        entry.confidence = 0.7
+      }
     }
 
-    // Priority 5: Filename heuristics (fallback)
+    // Priority 4: Filename heuristics (fallback)
     if (!entry.description) {
       entry.description = fileNameToDescription(baseName)
+      entry.source = "filename"
+      entry.confidence = Math.min(entry.confidence ?? 0.55, 0.55)
     }
     if (!entry.tags || entry.tags.length === 0) {
       entry.tags = extractTagsFromPath(file, dirPath)
     }
 
+    entry.tags = normalizeTerms(entry.tags ?? [])
+    entry.aliases = buildAliases(canonicalName, entry.tags)
+
+    // Intents are only generated when LLM is configured (via enhanceStashWithLlm)
+    // Heuristic intents are too noisy to be useful for search quality
+
     entry.entry = path.basename(file)
     entries.push(entry)
   }
@@ -145,6 +211,74 @@ export function generateMetadata(
   return { entries }
 }
 
+
+function normalizeTerms(values: string[]): string[] {
+  const normalized = new Set<string>()
+  for (const value of values) {
+    const cleaned = value.toLowerCase().replace(/[-_]+/g, " ").replace(/\s+/g, " ").trim()
+    if (!cleaned) continue
+    normalized.add(cleaned)
+    if (cleaned.endsWith("s") && cleaned.length > 3) {
+      normalized.add(cleaned.slice(0, -1))
+    }
+  }
+  return Array.from(normalized)
+}
+
+function buildAliases(name: string, tags: string[]): string[] {
+  const aliases = new Set<string>()
+  const spaced = name.replace(/[-_]+/g, " ").trim().toLowerCase()
+  if (spaced && spaced !== name.toLowerCase()) aliases.add(spaced)
+  if (tags.length > 1) aliases.add(tags.join(" "))
+  return Array.from(aliases)
+}
+
+// ── Intent Generation ────────────────────────────────────────────────────────
+
+export function generateIntents(description: string, tags: string[], name: string): string[] {
+  const intents = new Set<string>()
+
+  // Split name on separators to extract tokens and potential verb
+  const nameTokens = name
+    .replace(/[-_]+/g, " ")
+    .replace(/([a-z])([A-Z])/g, "$1 $2")
+    .toLowerCase()
+    .trim()
+    .split(/\s+/)
+    .filter((t) => t.length > 1)
+
+  // Intent from name as phrase (e.g. "summarize diff")
+  const namePhrase = nameTokens.join(" ")
+  if (namePhrase.length > 2) intents.add(namePhrase)
+
+  // Intent from description (lowercased)
+  const desc = description.toLowerCase().trim()
+  if (desc.length > 2) intents.add(desc)
+
+  // Combine first name token (potential verb) with tags
+  // e.g. name "summarize-diff", tags ["git"] → "summarize git diff"
+  if (nameTokens.length >= 1 && tags.length > 0) {
+    const verb = nameTokens[0]
+    const rest = nameTokens.slice(1).join(" ")
+    for (const tag of tags) {
+      const tagLower = tag.toLowerCase()
+      // verb + tag + rest (e.g. "summarize git diff")
+      const parts = [verb, tagLower, rest].filter((p) => p.length > 0)
+      const phrase = parts.join(" ")
+      if (phrase !== namePhrase && phrase.length > 2) intents.add(phrase)
+    }
+  }
+
+  // Join tag pairs (e.g. ["git", "diff"] → "git diff")
+  if (tags.length >= 2) {
+    const tagPhrase = tags.map((t) => t.toLowerCase()).join(" ")
+    if (tagPhrase.length > 2) intents.add(tagPhrase)
+  }
+
+  // Cap at 8 intents
+  return Array.from(intents).slice(0, 8)
+}
+
 export function extractDescriptionFromComments(filePath: string): string | null {
   let content: string
   try {
@@ -195,14 +329,8 @@ export function extractFrontmatterDescription(filePath: string): string | null {
     return null
   }
 
-  const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---/)
-  if (!match) return null
-
-  for (const line of match[1].split(/\r?\n/)) {
-    const m = line.match(/^description:\s*"?(.+?)"?\s*$/)
-    if (m) return m[1]
-  }
-  return null
+  const parsed = parseFrontmatter(content)
+  return toStringOrUndefined(parsed.data.description) ?? null
 }
 
 export function extractPackageMetadata(

package/src/ripgrep-install.ts

@@ -0,0 +1,200 @@
+import { spawnSync } from "node:child_process"
+import fs from "node:fs"
+import path from "node:path"
+import { IS_WINDOWS } from "./common"
+import { RG_BINARY, resolveRg } from "./ripgrep-resolve"
+
+/**
+ * Platform and architecture detection for ripgrep binary downloads.
+ */
+function getRgPlatformTarget(): { platform: string; arch: string; ext: string } | null {
+  const platform = process.platform
+  const arch = process.arch
+
+  if (platform === "linux" && arch === "x64") {
+    return { platform: "x86_64-unknown-linux-musl", arch: "x64", ext: ".tar.gz" }
+  }
+  if (platform === "linux" && arch === "arm64") {
+    return { platform: "aarch64-unknown-linux-gnu", arch: "arm64", ext: ".tar.gz" }
+  }
+  if (platform === "darwin" && arch === "x64") {
+    return { platform: "x86_64-apple-darwin", arch: "x64", ext: ".tar.gz" }
+  }
+  if (platform === "darwin" && arch === "arm64") {
+    return { platform: "aarch64-apple-darwin", arch: "arm64", ext: ".tar.gz" }
+  }
+  if (platform === "win32" && arch === "x64") {
+    return { platform: "x86_64-pc-windows-msvc", arch: "x64", ext: ".zip" }
+  }
+
+  return null
+}
+
+const RG_VERSION = "14.1.1"
+
+export interface EnsureRgResult {
+  rgPath: string
+  installed: boolean
+  version: string
+}
+
+/**
+ * Ensure ripgrep is available. If not found on PATH or in stash/bin,
+ * download and install it to stash/bin.
+ *
+ * Returns the path to the ripgrep binary and whether it was newly installed.
+ */
+export function ensureRg(stashDir: string): EnsureRgResult {
+  // Already available?
+  const existing = resolveRg(stashDir)
+  if (existing) {
+    return { rgPath: existing, installed: false, version: getRgVersion(existing) }
+  }
+
+  // Determine platform
+  const target = getRgPlatformTarget()
+  if (!target) {
+    throw new Error(
+      `Unsupported platform for ripgrep auto-install: ${process.platform}/${process.arch}. ` +
+      `Install ripgrep manually: https://github.com/BurntSushi/ripgrep#installation`
+    )
+  }
+
+  const binDir = path.join(stashDir, "bin")
+  if (!fs.existsSync(binDir)) {
+    fs.mkdirSync(binDir, { recursive: true })
+  }
+
+  const archiveName = `ripgrep-${RG_VERSION}-${target.platform}`
+  const url = `https://github.com/BurntSushi/ripgrep/releases/download/${RG_VERSION}/${archiveName}${target.ext}`
+  const destBinary = path.join(binDir, RG_BINARY)
+
+  if (target.ext === ".tar.gz") {
+    downloadAndExtractTarGz(url, archiveName, destBinary)
+  } else {
+    downloadAndExtractZip(url, archiveName, destBinary)
+  }
+
+  // Make executable
+  if (!IS_WINDOWS) {
+    fs.chmodSync(destBinary, 0o755)
+  }
+
+  return { rgPath: destBinary, installed: true, version: RG_VERSION }
+}
+
+function downloadAndExtractTarGz(url: string, archiveName: string, destBinary: string): void {
+  const destDir = path.dirname(destBinary)
+  const tmpTarGz = path.join(destDir, "rg-download.tar.gz")
+
+  try {
+    // Download archive to a temporary file without using a shell
+    const curlResult = spawnSync(
+      "curl",
+      ["-fsSL", "-o", tmpTarGz, url],
+      {
+        encoding: "utf8",
+        timeout: 60_000,
+      }
+    )
+
+    if (curlResult.status !== 0) {
+      const err = curlResult.stderr?.trim() || curlResult.error?.message || "unknown error"
+      throw new Error(`Failed to download ripgrep from ${url}: ${err}`)
+    }
+
+    // Extract the specific binary from the archive into destDir
+    const tarResult = spawnSync(
+      "tar",
+      [
+        "xzf",
+        tmpTarGz,
+        "--strip-components=1",
+        "-C",
+        destDir,
+        `${archiveName}/rg`,
+      ],
+      {
+        encoding: "utf8",
+        timeout: 60_000,
+      }
+    )
+
+    if (tarResult.status !== 0) {
+      const err = tarResult.stderr?.trim() || tarResult.error?.message || "unknown error"
+      throw new Error(`Failed to extract ripgrep from ${url}: ${err}`)
+    }
+
+    if (!fs.existsSync(destBinary)) {
+      throw new Error(`ripgrep binary not found at ${destBinary} after extraction`)
+    }
+  } finally {
+    // Best-effort cleanup of temporary archive
+    try {
+      if (fs.existsSync(tmpTarGz)) {
+        fs.unlinkSync(tmpTarGz)
+      }
+    } catch {
+      // ignore cleanup errors
+    }
+  }
+}
+
+function downloadAndExtractZip(url: string, archiveName: string, destBinary: string): void {
+  const destDir = path.dirname(destBinary)
+  const tmpZip = path.join(destDir, "rg-download.zip")
+  const expandedDir = path.join(destDir, archiveName)
+  try {
+    // Download
+    const dlResult = spawnSync("curl", ["-fsSL", "-o", tmpZip, url], {
+      encoding: "utf8",
+      timeout: 60_000,
+    })
+    if (dlResult.status !== 0) {
+      throw new Error(dlResult.stderr?.trim() || "download failed")
+    }
+
+    // Extract the zip archive using separate spawnSync calls with argument arrays
+    // to avoid shell injection via path interpolation in PowerShell -Command strings
+    const expandResult = spawnSync("powershell", [
+      "-Command",
+      "Expand-Archive",
+      "-Path", tmpZip,
+      "-DestinationPath", destDir,
+      "-Force",
+    ], {
+      encoding: "utf8",
+      timeout: 60_000,
+    })
+    if (expandResult.status !== 0) {
+      throw new Error(expandResult.stderr?.trim() || "extraction failed")
+    }
+
+    const srcRgExe = path.join(destDir, archiveName, "rg.exe")
+    const moveResult = spawnSync("powershell", [
+      "-Command",
+      "Move-Item",
+      "-Force",
+      "-Path", srcRgExe,
+      "-Destination", destBinary,
+    ], {
+      encoding: "utf8",
+      timeout: 60_000,
+    })
+    if (moveResult.status !== 0) {
+      throw new Error(moveResult.stderr?.trim() || "move failed")
+    }
+  } finally {
+    if (fs.existsSync(tmpZip)) fs.unlinkSync(tmpZip)
+    if (fs.existsSync(expandedDir)) fs.rmSync(expandedDir, { recursive: true, force: true })
+  }
+}
+
+function getRgVersion(rgPath: string): string {
+  const result = spawnSync(rgPath, ["--version"], { encoding: "utf8", timeout: 5_000 })
+  if (result.status === 0 && result.stdout) {
+    const match = result.stdout.match(/ripgrep\s+([\d.]+)/)
+    return match ? match[1] : "unknown"
+  }
+  return "unknown"
+}

package/src/ripgrep-resolve.ts

@@ -0,0 +1,72 @@
+import fs from "node:fs"
+import path from "node:path"
+import { IS_WINDOWS } from "./common"
+
+export const RG_BINARY = IS_WINDOWS ? "rg.exe" : "rg"
+
+function canExecute(filePath: string): boolean {
+  if (!fs.existsSync(filePath)) return false
+  if (IS_WINDOWS) return true
+  try {
+    fs.accessSync(filePath, fs.constants.X_OK)
+    return true
+  } catch {
+    return false
+  }
+}
+
+function resolveFromPath(): string | null {
+  const rawPath = process.env.PATH
+  if (!rawPath) return null
+
+  const pathEntries = rawPath.split(path.delimiter).filter(Boolean)
+
+  if (IS_WINDOWS) {
+    const pathext = (process.env.PATHEXT || ".EXE;.CMD;.BAT;.COM")
+      .split(";")
+      .filter(Boolean)
+      .map((ext) => ext.toLowerCase())
+
+    for (const entry of pathEntries) {
+      const directCandidate = path.join(entry, "rg")
+      if (canExecute(directCandidate)) return directCandidate
+
+      for (const ext of pathext) {
+        const candidate = path.join(entry, `rg${ext}`)
+        if (canExecute(candidate)) return candidate
+      }
+    }
+    return null
+  }
+
+  for (const entry of pathEntries) {
+    const candidate = path.join(entry, "rg")
+    if (canExecute(candidate)) return candidate
+  }
+
+  return null
+}
+
+/**
+ * Resolve the path to a usable ripgrep binary.
+ * Checks in order:
+ *   1. stashDir/bin/rg
+ *   2. system PATH (rg)
+ * Returns null if ripgrep is not available.
+ */
+export function resolveRg(stashDir?: string): string | null {
+  // Check stash bin directory first
+  if (stashDir) {
+    const stashRg = path.join(stashDir, "bin", RG_BINARY)
+    if (canExecute(stashRg)) return stashRg
+  }
+
+  return resolveFromPath()
+}
+
+/**
+ * Check if ripgrep is available (either in stash/bin or system PATH).
+ */
+export function isRgAvailable(stashDir?: string): boolean {
+  return resolveRg(stashDir) !== null
+}