agentikit 0.0.7 → 0.0.9

package/src/markdown.ts

@@ -0,0 +1,106 @@
+import { parseFrontmatter } from "./frontmatter"
+
+// ── Types ───────────────────────────────────────────────────────────────────
+
+export interface TocHeading {
+  level: number
+  text: string
+  line: number
+}
+
+export interface KnowledgeToc {
+  headings: TocHeading[]
+  totalLines: number
+}
+
+// ── Parsing ─────────────────────────────────────────────────────────────────
+
+export function parseMarkdownToc(content: string): KnowledgeToc {
+  const lines = content.split(/\r?\n/)
+  const headings: TocHeading[] = []
+
+  const parsed = parseFrontmatter(content)
+  const start = parsed.frontmatter ? parsed.bodyStartLine - 1 : 0
+
+  for (let i = start; i < lines.length; i++) {
+    const match = lines[i].match(/^(#{1,6})\s+(.+)$/)
+    if (match) {
+      headings.push({
+        level: match[1].length,
+        text: match[2].replace(/\s+#+\s*$/, "").trim(),
+        line: i + 1,
+      })
+    }
+  }
+
+  return { headings, totalLines: lines.length }
+}
+
+// ── Extraction ──────────────────────────────────────────────────────────────
+
+export function extractSection(
+  content: string,
+  heading: string,
+): { content: string; startLine: number; endLine: number } | null {
+  const lines = content.split(/\r?\n/)
+  const target = heading.toLowerCase()
+
+  let startIdx = -1
+  let startLevel = 0
+
+  for (let i = 0; i < lines.length; i++) {
+    const match = lines[i].match(/^(#{1,6})\s+(.+)$/)
+    if (!match) continue
+    const text = match[2].replace(/\s+#+\s*$/, "").trim()
+    if (text.toLowerCase() === target && startIdx === -1) {
+      startIdx = i
+      startLevel = match[1].length
+    } else if (startIdx !== -1 && match[1].length <= startLevel) {
+      return {
+        content: lines.slice(startIdx, i).join("\n"),
+        startLine: startIdx + 1,
+        endLine: i,
+      }
+    }
+  }
+
+  if (startIdx === -1) return null
+
+  return {
+    content: lines.slice(startIdx).join("\n"),
+    startLine: startIdx + 1,
+    endLine: lines.length,
+  }
+}
+
+export function extractLineRange(content: string, start: number, end: number): string {
+  const lines = content.split(/\r?\n/)
+  if (end < start) return ""
+  const s = Math.max(1, Math.min(start, lines.length))
+  const e = Math.min(end, lines.length)
+  return lines.slice(s - 1, e).join("\n")
+}
+
+export function extractFrontmatterOnly(content: string): string | null {
+  const parsed = parseFrontmatter(content)
+  return parsed.frontmatter
+}
+
+// ── Formatting ──────────────────────────────────────────────────────────────
+
+export function formatToc(toc: KnowledgeToc): string {
+  if (toc.headings.length === 0) {
+    return `(no headings found — ${toc.totalLines} lines total)`
+  }
+
+  const lineWidth = String(toc.totalLines).length
+  const parts = toc.headings.map((h) => {
+    const lineNum = `L${String(h.line).padStart(lineWidth)}`
+    const indent = "  ".repeat(h.level - 1)
+    const prefix = "#".repeat(h.level)
+    return `${lineNum}  ${indent}${prefix} ${h.text}`
+  })
+
+  parts.push(`\n${toc.totalLines} lines total`)
+  return parts.join("\n")
+}

package/src/metadata.ts

@@ -1,6 +1,9 @@
 import fs from "node:fs"
 import path from "node:path"
-import type { AgentikitAssetType } from "./stash"
+import { type AgentikitAssetType, isAssetType } from "./common"
+import { SCRIPT_EXTENSIONS, isRelevantAssetFile, deriveCanonicalAssetName } from "./asset-spec"
+import { parseFrontmatter, toStringOrUndefined } from "./frontmatter"
+import { parseMarkdownToc, type TocHeading } from "./markdown"
 
 // ── Schema ──────────────────────────────────────────────────────────────────
 
@@ -16,9 +19,16 @@ export interface StashEntry {
   description?: string
   tags?: string[]
   examples?: string[]
+  intents?: string[]
   intent?: StashIntent
   entry?: string
   generated?: boolean
+  quality?: "generated" | "curated"
+  confidence?: number
+  source?: "package" | "frontmatter" | "comments" | "filename" | "manual" | "llm"
+  aliases?: string[]
+  toc?: TocHeading[]
+  usage?: string[]
 }
 
 export interface StashFile {
@@ -59,7 +69,7 @@ export function validateStashEntry(entry: unknown): StashEntry | null {
   if (typeof entry !== "object" || entry === null) return null
   const e = entry as Record<string, unknown>
   if (typeof e.name !== "string" || !e.name) return null
-  if (typeof e.type !== "string" || !isValidType(e.type)) return null
+  if (typeof e.type !== "string" || !isAssetType(e.type)) return null
 
   const result: StashEntry = {
     name: e.name,
@@ -68,6 +78,10 @@ export function validateStashEntry(entry: unknown): StashEntry | null {
   if (typeof e.description === "string" && e.description) result.description = e.description
   if (Array.isArray(e.tags)) result.tags = e.tags.filter((t): t is string => typeof t === "string")
   if (Array.isArray(e.examples)) result.examples = e.examples.filter((x): x is string => typeof x === "string")
+  if (Array.isArray(e.intents)) {
+    const filtered = e.intents.filter((s): s is string => typeof s === "string" && s.trim().length > 0)
+    if (filtered.length > 0) result.intents = filtered
+  }
   if (typeof e.intent === "object" && e.intent !== null) {
     const intent = e.intent as Record<string, unknown>
     result.intent = {}
@@ -77,67 +91,135 @@ export function validateStashEntry(entry: unknown): StashEntry | null {
   }
   if (typeof e.entry === "string" && e.entry) result.entry = e.entry
   if (e.generated === true) result.generated = true
+  if (e.quality === "generated" || e.quality === "curated") result.quality = e.quality
+  if (typeof e.confidence === "number" && Number.isFinite(e.confidence)) result.confidence = Math.max(0, Math.min(1, e.confidence))
+  if (typeof e.source === "string" && ["package", "frontmatter", "comments", "filename", "manual", "llm"].includes(e.source)) {
+    result.source = e.source as StashEntry["source"]
+  }
+  if (Array.isArray(e.aliases)) {
+    const filtered = e.aliases.filter((a): a is string => typeof a === "string" && a.trim().length > 0)
+    if (filtered.length > 0) result.aliases = normalizeTerms(filtered)
+  }
+  if (Array.isArray(e.toc)) {
+    const validated = e.toc.filter(
+      (h: unknown): h is TocHeading => {
+        if (typeof h !== "object" || h === null) return false
+        const rec = h as Record<string, unknown>
+        return typeof rec.level === "number"
+          && typeof rec.text === "string"
+          && typeof rec.line === "number"
+      },
+    )
+    if (validated.length > 0) result.toc = validated
+  }
+  const usage = normalizeNonEmptyStringList(e.usage)
+  if (usage) result.usage = usage
 
   return result
 }
 
-function isValidType(type: string): boolean {
-  return type === "tool" || type === "skill" || type === "command" || type === "agent"
+function normalizeNonEmptyStringList(value: unknown): string[] | undefined {
+  if (typeof value === "string") {
+    const trimmed = value.trim()
+    return trimmed ? [trimmed] : undefined
+  }
+  if (!Array.isArray(value)) return undefined
+  const filtered = value
+    .filter((item): item is string => typeof item === "string")
+    .map((item) => item.trim())
+    .filter((item) => item.length > 0)
+  return filtered.length > 0 ? filtered : undefined
 }
 
 // ── Metadata Generation ─────────────────────────────────────────────────────
 
-const SCRIPT_EXTENSIONS = new Set([".sh", ".ts", ".js", ".ps1", ".cmd", ".bat"])
-
 export function generateMetadata(
   dirPath: string,
   assetType: AgentikitAssetType,
   files: string[],
+  typeRoot = dirPath,
 ): StashFile {
   const entries: StashEntry[] = []
+  const pkgMeta = extractPackageMetadata(dirPath)
 
   for (const file of files) {
     const ext = path.extname(file).toLowerCase()
     const baseName = path.basename(file, ext)
+    const fileName = path.basename(file)
 
     // Skip non-relevant files
-    if (assetType === "tool" && !SCRIPT_EXTENSIONS.has(ext)) continue
-    if ((assetType === "command" || assetType === "agent") && ext !== ".md") continue
-    if (assetType === "skill" && path.basename(file) !== "SKILL.md") continue
+    if (!isRelevantAssetFile(assetType, fileName)) continue
+
+    const canonicalName = assetType === "skill"
+      ? deriveCanonicalAssetName(assetType, typeRoot, file) ?? baseName
+      : baseName
 
     const entry: StashEntry = {
-      name: baseName,
+      name: canonicalName,
       type: assetType,
       generated: true,
+      quality: "generated",
+      confidence: 0.55,
+      source: "filename",
     }
 
-    // Priority 3: package.json metadata
-    const pkgMeta = extractPackageMetadata(dirPath)
+    // Priority 1: package.json metadata
     if (pkgMeta) {
-      if (pkgMeta.description && !entry.description) entry.description = pkgMeta.description
-      if (pkgMeta.keywords && pkgMeta.keywords.length > 0) entry.tags = pkgMeta.keywords
+      if (pkgMeta.description && !entry.description) {
+        entry.description = pkgMeta.description
+        entry.source = "package"
+        entry.confidence = 0.8
+      }
+      if (pkgMeta.keywords && pkgMeta.keywords.length > 0) entry.tags = normalizeTerms(pkgMeta.keywords)
     }
 
-    // Priority 2: Frontmatter (for .md files)
+    // Priority 2: Frontmatter (for .md files — overrides package.json description)
     if (ext === ".md") {
       const fm = extractFrontmatterDescription(file)
-      if (fm) entry.description = fm
+      if (fm) {
+        entry.description = fm
+        entry.source = "frontmatter"
+        entry.confidence = 0.9
+      }
+    }
+
+    // Knowledge entries: generate TOC from headings
+    if (assetType === "knowledge") {
+      try {
+        const mdContent = fs.readFileSync(file, "utf8")
+        const toc = parseMarkdownToc(mdContent)
+        if (toc.headings.length > 0) entry.toc = toc.headings
+      } catch {
+        // Non-fatal: skip TOC if file can't be read
+      }
     }
 
-    // Priority 4: Code comments (for script files)
+    // Priority 3: Code comments (for script files)
     if (SCRIPT_EXTENSIONS.has(ext) && ext !== ".md") {
       const commentDesc = extractDescriptionFromComments(file)
-      if (commentDesc && !entry.description) entry.description = commentDesc
+      if (commentDesc && !entry.description) {
+        entry.description = commentDesc
+        entry.source = "comments"
+        entry.confidence = 0.7
+      }
     }
 
-    // Priority 5: Filename heuristics (fallback)
+    // Priority 4: Filename heuristics (fallback)
     if (!entry.description) {
       entry.description = fileNameToDescription(baseName)
+      entry.source = "filename"
+      entry.confidence = Math.min(entry.confidence ?? 0.55, 0.55)
     }
     if (!entry.tags || entry.tags.length === 0) {
       entry.tags = extractTagsFromPath(file, dirPath)
     }
 
+    entry.tags = normalizeTerms(entry.tags ?? [])
+    entry.aliases = buildAliases(canonicalName, entry.tags)
+
+    // Intents are only generated when LLM is configured (via enhanceStashWithLlm)
+    // Heuristic intents are too noisy to be useful for search quality
+
     entry.entry = path.basename(file)
     entries.push(entry)
   }
@@ -145,6 +227,74 @@ export function generateMetadata(
   return { entries }
 }
 
+
+function normalizeTerms(values: string[]): string[] {
+  const normalized = new Set<string>()
+  for (const value of values) {
+    const cleaned = value.toLowerCase().replace(/[-_]+/g, " ").replace(/\s+/g, " ").trim()
+    if (!cleaned) continue
+    normalized.add(cleaned)
+    if (cleaned.endsWith("s") && cleaned.length > 3) {
+      normalized.add(cleaned.slice(0, -1))
+    }
+  }
+  return Array.from(normalized)
+}
+
+function buildAliases(name: string, tags: string[]): string[] {
+  const aliases = new Set<string>()
+  const spaced = name.replace(/[-_]+/g, " ").trim().toLowerCase()
+  if (spaced && spaced !== name.toLowerCase()) aliases.add(spaced)
+  if (tags.length > 1) aliases.add(tags.join(" "))
+  return Array.from(aliases)
+}
+
+// ── Intent Generation ────────────────────────────────────────────────────────
+
+export function generateIntents(description: string, tags: string[], name: string): string[] {
+  const intents = new Set<string>()
+
+  // Split name on separators to extract tokens and potential verb
+  const nameTokens = name
+    .replace(/[-_]+/g, " ")
+    .replace(/([a-z])([A-Z])/g, "$1 $2")
+    .toLowerCase()
+    .trim()
+    .split(/\s+/)
+    .filter((t) => t.length > 1)
+
+  // Intent from name as phrase (e.g. "summarize diff")
+  const namePhrase = nameTokens.join(" ")
+  if (namePhrase.length > 2) intents.add(namePhrase)
+
+  // Intent from description (lowercased)
+  const desc = description.toLowerCase().trim()
+  if (desc.length > 2) intents.add(desc)
+
+  // Combine first name token (potential verb) with tags
+  // e.g. name "summarize-diff", tags ["git"] → "summarize git diff"
+  if (nameTokens.length >= 1 && tags.length > 0) {
+    const verb = nameTokens[0]
+    const rest = nameTokens.slice(1).join(" ")
+    for (const tag of tags) {
+      const tagLower = tag.toLowerCase()
+      // verb + tag + rest (e.g. "summarize git diff")
+      const parts = [verb, tagLower, rest].filter((p) => p.length > 0)
+      const phrase = parts.join(" ")
+      if (phrase !== namePhrase && phrase.length > 2) intents.add(phrase)
+    }
+  }
+
+  // Join tag pairs (e.g. ["git", "diff"] → "git diff")
+  if (tags.length >= 2) {
+    const tagPhrase = tags.map((t) => t.toLowerCase()).join(" ")
+    if (tagPhrase.length > 2) intents.add(tagPhrase)
+  }
+
+  // Cap at 8 intents
+  return Array.from(intents).slice(0, 8)
+}
+
 export function extractDescriptionFromComments(filePath: string): string | null {
   let content: string
   try {
@@ -195,14 +345,8 @@ export function extractFrontmatterDescription(filePath: string): string | null {
     return null
   }
 
-  const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---/)
-  if (!match) return null
-
-  for (const line of match[1].split(/\r?\n/)) {
-    const m = line.match(/^description:\s*"?(.+?)"?\s*$/)
-    if (m) return m[1]
-  }
-  return null
+  const parsed = parseFrontmatter(content)
+  return toStringOrUndefined(parsed.data.description) ?? null
 }
 
 export function extractPackageMetadata(

package/src/registry-install.ts

@@ -0,0 +1,245 @@
+import { spawnSync } from "node:child_process"
+import fs from "node:fs"
+import path from "node:path"
+import { TYPE_DIRS } from "./common"
+import { loadConfig, saveConfig, type AgentikitConfig } from "./config"
+import { parseRegistryRef, resolveRegistryArtifact } from "./registry-resolve"
+import type { RegistryInstallResult, RegistryInstalledEntry, RegistrySource } from "./registry-types"
+
+const REGISTRY_STASH_DIR_NAMES = new Set<string>(Object.values(TYPE_DIRS))
+
+export interface InstallRegistryRefOptions {
+  cacheRootDir?: string
+  now?: Date
+}
+
+export async function installRegistryRef(ref: string, options?: InstallRegistryRefOptions): Promise<RegistryInstallResult> {
+  const parsed = parseRegistryRef(ref)
+  const resolved = await resolveRegistryArtifact(parsed)
+
+  const installedAt = (options?.now ?? new Date()).toISOString()
+  const cacheRootDir = options?.cacheRootDir ?? getRegistryCacheRootDir()
+  const cacheDir = buildInstallCacheDir(cacheRootDir, resolved.source, resolved.id)
+  const archivePath = path.join(cacheDir, "artifact.tar.gz")
+  const extractedDir = path.join(cacheDir, "extracted")
+
+  fs.mkdirSync(cacheDir, { recursive: true })
+
+  await downloadArchive(resolved.artifactUrl, archivePath)
+  extractTarGzSecure(archivePath, extractedDir)
+
+  const stashRoot = detectStashRoot(extractedDir)
+
+  return {
+    id: resolved.id,
+    source: resolved.source,
+    ref: resolved.ref,
+    artifactUrl: resolved.artifactUrl,
+    resolvedVersion: resolved.resolvedVersion,
+    resolvedRevision: resolved.resolvedRevision,
+    installedAt,
+    cacheDir,
+    extractedDir,
+    stashRoot,
+  }
+}
+
+export function upsertInstalledRegistryEntry(entry: RegistryInstalledEntry, stashDir?: string): AgentikitConfig {
+  const current = loadConfig(stashDir)
+  const currentInstalled = current.registry?.installed ?? []
+  const previousRegistryRoots = new Set(currentInstalled.map((item) => path.resolve(item.stashRoot)))
+
+  const withoutExisting = currentInstalled.filter((item) => item.id !== entry.id)
+  const nextInstalled = [...withoutExisting, normalizeInstalledEntry(entry)]
+  const nextRegistryRoots = new Set(nextInstalled.map((item) => path.resolve(item.stashRoot)))
+  const preservedAdditional = current.additionalStashDirs.filter(
+    (dir) => !previousRegistryRoots.has(path.resolve(dir)),
+  )
+  const syncedAdditional = uniquePaths([...preservedAdditional, ...nextRegistryRoots])
+
+  const nextConfig: AgentikitConfig = {
+    ...current,
+    additionalStashDirs: syncedAdditional,
+    registry: {
+      installed: nextInstalled,
+    },
+  }
+  saveConfig(nextConfig, stashDir)
+  return nextConfig
+}
+
+export function removeInstalledRegistryEntry(id: string, stashDir?: string): AgentikitConfig {
+  const current = loadConfig(stashDir)
+  const currentInstalled = current.registry?.installed ?? []
+  const previousRegistryRoots = new Set(currentInstalled.map((item) => path.resolve(item.stashRoot)))
+
+  const nextInstalled = currentInstalled.filter((item) => item.id !== id)
+  const nextRegistryRoots = new Set(nextInstalled.map((item) => path.resolve(item.stashRoot)))
+
+  const preservedAdditional = current.additionalStashDirs.filter(
+    (dir) => !previousRegistryRoots.has(path.resolve(dir)),
+  )
+  const syncedAdditional = uniquePaths([...preservedAdditional, ...nextRegistryRoots])
+
+  const nextConfig: AgentikitConfig = {
+    ...current,
+    additionalStashDirs: syncedAdditional,
+    registry: nextInstalled.length > 0 ? { installed: nextInstalled } : undefined,
+  }
+  saveConfig(nextConfig, stashDir)
+  return nextConfig
+}
+
+export function getRegistryCacheRootDir(): string {
+  const xdgCache = process.env.XDG_CACHE_HOME?.trim()
+  if (xdgCache) {
+    return path.join(path.resolve(xdgCache), "agentikit", "registry")
+  }
+  const home = process.env.HOME?.trim()
+  if (!home) {
+    throw new Error("Unable to determine cache directory. Set XDG_CACHE_HOME or HOME.")
+  }
+  return path.join(path.resolve(home), ".cache", "agentikit", "registry")
+}
+
+export function detectStashRoot(extractedDir: string): string {
+  const root = path.resolve(extractedDir)
+
+  const rootDotStash = path.join(root, ".stash")
+  if (isDirectory(rootDotStash)) {
+    return root
+  }
+
+  if (hasStashDirs(root)) {
+    return root
+  }
+
+  const opencodeDir = path.join(root, "opencode")
+  if (hasStashDirs(opencodeDir)) {
+    return opencodeDir
+  }
+
+  const shallowest = findShallowestDotStashRoot(root)
+  if (shallowest) return shallowest
+
+  return root
+}
+
+function buildInstallCacheDir(cacheRootDir: string, source: RegistrySource, id: string): string {
+  const slug = `${source}-${id.replace(/[^a-zA-Z0-9_.-]+/g, "-").replace(/^-+|-+$/g, "")}`
+  const stamp = `${Date.now()}-${Math.random().toString(36).slice(2, 10)}`
+  return path.join(cacheRootDir, slug || source, stamp)
+}
+
+async function downloadArchive(url: string, destination: string): Promise<void> {
+  const response = await fetch(url)
+  if (!response.ok) {
+    throw new Error(`Failed to download archive (${response.status}) from ${url}`)
+  }
+  const arrayBuffer = await response.arrayBuffer()
+  fs.writeFileSync(destination, Buffer.from(arrayBuffer))
+}
+
+function extractTarGzSecure(archivePath: string, destinationDir: string): void {
+  const listResult = spawnSync("tar", ["tzf", archivePath], { encoding: "utf8" })
+  if (listResult.status !== 0) {
+    const err = listResult.stderr?.trim() || listResult.error?.message || "unknown error"
+    throw new Error(`Failed to inspect archive ${archivePath}: ${err}`)
+  }
+
+  validateTarEntries(listResult.stdout)
+
+  fs.rmSync(destinationDir, { recursive: true, force: true })
+  fs.mkdirSync(destinationDir, { recursive: true })
+
+  const extractResult = spawnSync("tar", ["xzf", archivePath, "--strip-components=1", "-C", destinationDir], {
+    encoding: "utf8",
+  })
+  if (extractResult.status !== 0) {
+    const err = extractResult.stderr?.trim() || extractResult.error?.message || "unknown error"
+    throw new Error(`Failed to extract archive ${archivePath}: ${err}`)
+  }
+}
+
+function validateTarEntries(listOutput: string): void {
+  const lines = listOutput.split(/\r?\n/).filter(Boolean)
+  for (const rawLine of lines) {
+    const entry = rawLine.trim()
+    if (!entry || entry.includes("\0")) {
+      throw new Error(`Archive contains an invalid entry: ${JSON.stringify(rawLine)}`)
+    }
+    if (entry.startsWith("/")) {
+      throw new Error(`Archive contains an absolute path entry: ${entry}`)
+    }
+
+    const normalized = path.posix.normalize(entry)
+    if (normalized === ".." || normalized.startsWith("../")) {
+      throw new Error(`Archive contains a path traversal entry: ${entry}`)
+    }
+
+    const parts = normalized.split("/").filter(Boolean)
+    const stripped = parts.slice(1).join("/")
+    if (!stripped) continue
+    const normalizedStripped = path.posix.normalize(stripped)
+    if (normalizedStripped === ".." || normalizedStripped.startsWith("../") || path.posix.isAbsolute(normalizedStripped)) {
+      throw new Error(`Archive contains an unsafe entry after strip-components: ${entry}`)
+    }
+  }
+}
+
+function isDirectory(target: string): boolean {
+  try {
+    return fs.statSync(target).isDirectory()
+  } catch {
+    return false
+  }
+}
+
+function hasStashDirs(dirPath: string): boolean {
+  if (!isDirectory(dirPath)) return false
+  const entries = fs.readdirSync(dirPath, { withFileTypes: true })
+  return entries.some((entry) => entry.isDirectory() && REGISTRY_STASH_DIR_NAMES.has(entry.name))
+}
+
+function findShallowestDotStashRoot(root: string): string | undefined {
+  const queue: string[] = [root]
+  while (queue.length > 0) {
+    const current = queue.shift()!
+    const dotStash = path.join(current, ".stash")
+    if (isDirectory(dotStash)) {
+      return current
+    }
+    let children: fs.Dirent[]
+    try {
+      children = fs.readdirSync(current, { withFileTypes: true })
+    } catch {
+      continue
+    }
+    for (const child of children) {
+      if (!child.isDirectory()) continue
+      if (child.name === ".git" || child.name === "node_modules") continue
+      queue.push(path.join(current, child.name))
+    }
+  }
+  return undefined
+}
+
+function normalizeInstalledEntry(entry: RegistryInstalledEntry): RegistryInstalledEntry {
+  return {
+    ...entry,
+    stashRoot: path.resolve(entry.stashRoot),
+    cacheDir: path.resolve(entry.cacheDir),
+  }
+}
+
+function uniquePaths(paths: Iterable<string>): string[] {
+  const seen = new Set<string>()
+  const result: string[] = []
+  for (const candidate of paths) {
+    const normalized = path.resolve(candidate)
+    if (seen.has(normalized)) continue
+    seen.add(normalized)
+    result.push(normalized)
+  }
+  return result
+}