agentikit 0.0.3 → 0.0.8

package/src/indexer.ts

@@ -1,6 +1,7 @@
 import fs from "node:fs"
 import path from "node:path"
-import type { AgentikitAssetType } from "./stash"
+import { type AgentikitAssetType, resolveStashDir } from "./common"
+import { ASSET_TYPES, TYPE_DIRS, deriveCanonicalAssetName } from "./asset-spec"
 import {
   type StashFile,
   type StashEntry,
@@ -8,7 +9,10 @@ import {
   writeStashFile,
   generateMetadata,
 } from "./metadata"
-import { TfIdfAdapter, type ScoredEntry } from "./similarity"
+import { TfIdfAdapter, type ScoredEntry, type SerializedTfIdf } from "./similarity"
+import { walkStash } from "./walker"
+import type { EmbeddingVector } from "./embedder"
+import type { LlmConnectionConfig } from "./config"
 
 // ── Types ───────────────────────────────────────────────────────────────────
 
@@ -16,15 +20,20 @@ export interface IndexedEntry {
   entry: StashEntry
   path: string
   dirPath: string
+  embedding?: EmbeddingVector
 }
 
 export interface SearchIndex {
   version: number
   builtAt: string
   stashDir: string
+  /** All stash directories that were indexed (primary + additional) */
+  stashDirs?: string[]
   entries: IndexedEntry[]
   /** Serialized TF-IDF state (term frequencies, idf values) */
-  tfidf?: unknown
+  tfidf?: SerializedTfIdf
+  /** Whether embeddings are included in entries */
+  hasEmbeddings?: boolean
 }
 
 export interface IndexResponse {
@@ -32,19 +41,16 @@ export interface IndexResponse {
   totalEntries: number
   generatedMetadata: number
   indexPath: string
+  mode: "full" | "incremental"
+  directoriesScanned: number
+  directoriesSkipped: number
+  /** Timing counters in milliseconds */
+  timing?: { totalMs: number; walkMs: number; embedMs: number; tfidfMs: number }
 }
 
 // ── Constants ───────────────────────────────────────────────────────────────
 
-const INDEX_VERSION = 1
-const SCRIPT_EXTENSIONS = new Set([".sh", ".ts", ".js", ".ps1", ".cmd", ".bat"])
-
-const TYPE_DIRS: Record<AgentikitAssetType, string> = {
-  tool: "tools",
-  skill: "skills",
-  command: "commands",
-  agent: "agents",
-}
+const INDEX_VERSION = 4
 
 // ── Index Path ──────────────────────────────────────────────────────────────
 
@@ -68,42 +74,109 @@ export function loadSearchIndex(): SearchIndex | null {
 
 // ── Indexer ──────────────────────────────────────────────────────────────────
 
-export function agentikitIndex(options?: { stashDir?: string }): IndexResponse {
-  const stashDir = options?.stashDir || resolveStashDirForIndex()
+export async function agentikitIndex(options?: { stashDir?: string; full?: boolean }): Promise<IndexResponse> {
+  const stashDir = options?.stashDir || resolveStashDir()
+
+  // Load config to get additional stash dirs and semantic search setting
+  const { loadConfig } = await import("./config.js")
+  const config = loadConfig(stashDir)
+
+  const allStashDirs = [stashDir]
+  for (const d of config.additionalStashDirs) {
+    try {
+      if (fs.statSync(d).isDirectory() && !allStashDirs.includes(path.resolve(d))) {
+        allStashDirs.push(path.resolve(d))
+      }
+    } catch { /* skip nonexistent dirs */ }
+  }
+
+  const t0 = Date.now()
   const allEntries: IndexedEntry[] = []
   let generatedCount = 0
+  let scannedDirs = 0
+  let skippedDirs = 0
 
-  for (const assetType of Object.keys(TYPE_DIRS) as AgentikitAssetType[]) {
-    const typeRoot = path.join(stashDir, TYPE_DIRS[assetType])
-    if (!fs.existsSync(typeRoot) || !fs.statSync(typeRoot).isDirectory()) continue
+  // Load previous index for incremental mode
+  const previousIndex = !options?.full ? loadSearchIndex() : null
+  const isIncremental = previousIndex !== null && previousIndex.stashDir === stashDir
+  const builtAtMs = isIncremental ? new Date(previousIndex.builtAt).getTime() : 0
 
-    // Group files by their immediate parent directory
-    const dirGroups = collectDirectoryGroups(typeRoot, assetType)
+  // Build lookup of previous entries by dirPath
+  const previousEntriesByDir = new Map<string, IndexedEntry[]>()
+  if (isIncremental) {
+    for (const ie of previousIndex.entries) {
+      const list = previousEntriesByDir.get(ie.dirPath) || []
+      list.push(ie)
+      previousEntriesByDir.set(ie.dirPath, list)
+    }
+  }
+
+  const seenPaths = new Set<string>()
+  const tWalkStart = Date.now()
+
+  for (const currentStashDir of allStashDirs) {
+    for (const assetType of ASSET_TYPES as AgentikitAssetType[]) {
+      const typeRoot = path.join(currentStashDir, TYPE_DIRS[assetType])
+      try {
+        if (!fs.statSync(typeRoot).isDirectory()) continue
+      } catch { continue }
+
+      // Group files by their immediate parent directory
+      const dirGroups = walkStash(typeRoot, assetType)
 
-    for (const [dirPath, files] of dirGroups) {
-      // Try loading existing .stash.json
-      let stash = loadStashFile(dirPath)
+      for (const { dirPath, files } of dirGroups) {
+        // Deduplicate by dirPath across stash dirs
+        if (seenPaths.has(path.resolve(dirPath))) continue
+        seenPaths.add(path.resolve(dirPath))
 
-      if (!stash) {
-        // Generate metadata
-        stash = generateMetadata(dirPath, assetType, files)
-        if (stash.entries.length > 0) {
-          writeStashFile(dirPath, stash)
-          generatedCount += stash.entries.length
+        // Incremental: skip directories that haven't changed
+        const prevEntries = previousEntriesByDir.get(dirPath)
+        if (isIncremental && prevEntries && !isDirStale(dirPath, files, prevEntries, builtAtMs)) {
+          allEntries.push(...prevEntries)
+          skippedDirs++
+          continue
         }
-      }
 
-      if (stash) {
-        for (const entry of stash.entries) {
-          const entryPath = entry.entry
-            ? path.join(dirPath, entry.entry)
-            : files[0] || dirPath
-          allEntries.push({ entry, path: entryPath, dirPath })
+        scannedDirs++
+
+        // Try loading existing .stash.json
+        let stash = loadStashFile(dirPath)
+
+        if (stash) {
+          const migration = migrateGeneratedSkillMetadata(stash, files, typeRoot)
+          if (migration.changed) {
+            stash = migration.stash
+            writeStashFile(dirPath, stash)
+          }
+        }
+
+        if (!stash) {
+          // Generate metadata
+          stash = generateMetadata(dirPath, assetType, files, typeRoot)
+          // Enhance with LLM if configured
+          if (config.llm && stash.entries.length > 0) {
+            stash = await enhanceStashWithLlm(config.llm, stash, dirPath, files)
+          }
+          if (stash.entries.length > 0) {
+            writeStashFile(dirPath, stash)
+            generatedCount += stash.entries.length
+          }
+        }
+
+        if (stash) {
+          for (const entry of stash.entries) {
+            const entryPath = entry.entry
+              ? path.join(dirPath, entry.entry)
+              : files[0] || dirPath
+            allEntries.push({ entry, path: entryPath, dirPath })
+          }
         }
       }
     }
   }
 
+  const tWalkEnd = Date.now()
+
   // Build TF-IDF index
   const adapter = new TfIdfAdapter()
   const scoredEntries: ScoredEntry[] = allEntries.map((ie) => ({
@@ -113,6 +186,26 @@ export function agentikitIndex(options?: { stashDir?: string }): IndexResponse {
     path: ie.path,
   }))
   adapter.buildIndex(scoredEntries)
+  const tTfidfEnd = Date.now()
+
+  // Generate embeddings if semantic search is enabled
+  let hasEmbeddings = false
+  if (config.semanticSearch) {
+    try {
+      const { embed } = await import("./embedder.js")
+      for (const ie of allEntries) {
+        if (!ie.embedding) {
+          const text = buildSearchText(ie.entry)
+          ie.embedding = await embed(text, config.embedding)
+        }
+      }
+      hasEmbeddings = true
+    } catch {
+      // Embedding provider not available, continue without embeddings
+    }
+  }
+
+  const tEmbedEnd = Date.now()
 
   // Persist index
   const indexPath = getIndexPath()
@@ -125,64 +218,138 @@ export function agentikitIndex(options?: { stashDir?: string }): IndexResponse {
     version: INDEX_VERSION,
     builtAt: new Date().toISOString(),
     stashDir,
+    stashDirs: allStashDirs,
     entries: allEntries,
     tfidf: adapter.serialize(),
+    hasEmbeddings,
   }
   fs.writeFileSync(indexPath, JSON.stringify(index) + "\n", "utf8")
 
+  const tEnd = Date.now()
+
   return {
     stashDir,
     totalEntries: allEntries.length,
     generatedMetadata: generatedCount,
     indexPath,
+    mode: isIncremental ? "incremental" : "full",
+    directoriesScanned: scannedDirs,
+    directoriesSkipped: skippedDirs,
+    timing: {
+      totalMs: tEnd - t0,
+      walkMs: tWalkEnd - tWalkStart, // includes metadata generation (interleaved)
+      embedMs: tEmbedEnd - tTfidfEnd,
+      tfidfMs: tTfidfEnd - tWalkEnd,
+    },
   }
 }
 
 // ── Helpers ─────────────────────────────────────────────────────────────────
 
-function collectDirectoryGroups(
-  typeRoot: string,
-  assetType: AgentikitAssetType,
-): Map<string, string[]> {
-  const groups = new Map<string, string[]>()
-
-  const walk = (dir: string): void => {
-    if (!fs.existsSync(dir)) return
-    const entries = fs.readdirSync(dir, { withFileTypes: true })
-    for (const entry of entries) {
-      if (entry.name === ".stash.json") continue
-      const fullPath = path.join(dir, entry.name)
-      if (entry.isDirectory()) {
-        walk(fullPath)
-      } else if (entry.isFile() && isRelevantFile(entry.name, assetType)) {
-        const parentDir = path.dirname(fullPath)
-        const existing = groups.get(parentDir)
-        if (existing) {
-          existing.push(fullPath)
-        } else {
-          groups.set(parentDir, [fullPath])
-        }
-      }
+function isDirStale(
+  dirPath: string,
+  currentFiles: string[],
+  previousEntries: IndexedEntry[],
+  builtAtMs: number,
+): boolean {
+  // Check if file set changed (additions or deletions)
+  const prevFileNames = new Set(
+    previousEntries
+      .map((ie) => ie.entry.entry)
+      .filter((e): e is string => !!e),
+  )
+  const currFileNames = new Set(currentFiles.map((f) => path.basename(f)))
+  if (prevFileNames.size !== currFileNames.size) return true
+  for (const name of currFileNames) {
+    if (!prevFileNames.has(name)) return true
+  }
+
+  // Check modification times of current files
+  for (const file of currentFiles) {
+    try {
+      if (fs.statSync(file).mtimeMs > builtAtMs) return true
+    } catch {
+      return true
     }
   }
 
-  walk(typeRoot)
-  return groups
+  // Check .stash.json modification time
+  const stashPath = path.join(dirPath, ".stash.json")
+  try {
+    if (fs.existsSync(stashPath) && fs.statSync(stashPath).mtimeMs > builtAtMs) return true
+  } catch {
+    // ignore
+  }
+
+  return false
+}
+
+function migrateGeneratedSkillMetadata(
+  stash: StashFile,
+  files: string[],
+  typeRoot: string,
+): { stash: StashFile; changed: boolean } {
+  const fileByBaseName = new Map(files.map((filePath) => [path.basename(filePath), filePath]))
+  let changed = false
+
+  const entries = stash.entries.map((entry) => {
+    if (entry.type !== "skill" || entry.generated !== true) return entry
+
+    const hintedFilePath = entry.entry ? fileByBaseName.get(path.basename(entry.entry)) : undefined
+    const skillFilePath = hintedFilePath ?? fileByBaseName.get("SKILL.md")
+    if (!skillFilePath) return entry
+
+    const canonicalName = deriveCanonicalAssetName("skill", typeRoot, skillFilePath)
+    if (!canonicalName || canonicalName === entry.name) return entry
+
+    changed = true
+    return { ...entry, name: canonicalName }
+  })
+
+  if (!changed) {
+    return { stash, changed: false }
+  }
+
+  return {
+    stash: { entries },
+    changed: true,
+  }
 }
 
-function isRelevantFile(fileName: string, assetType: AgentikitAssetType): boolean {
-  const ext = path.extname(fileName).toLowerCase()
-  switch (assetType) {
-    case "tool":
-      return SCRIPT_EXTENSIONS.has(ext)
-    case "skill":
-      return fileName === "SKILL.md"
-    case "command":
-    case "agent":
-      return ext === ".md"
-    default:
-      return false
+async function enhanceStashWithLlm(
+  llmConfig: LlmConnectionConfig,
+  stash: StashFile,
+  dirPath: string,
+  files: string[],
+): Promise<StashFile> {
+  const { enhanceMetadata } = await import("./llm.js")
+
+  const enhanced: StashEntry[] = []
+  for (const entry of stash.entries) {
+    try {
+      // Find the file matching this entry for content context
+      const entryFile = entry.entry
+        ? files.find((f) => path.basename(f) === entry.entry) ?? files[0]
+        : files[0]
+      let fileContent: string | undefined
+      if (entryFile) {
+        try {
+          fileContent = fs.readFileSync(entryFile, "utf8")
+        } catch { /* ignore unreadable files */ }
+      }
+
+      const improvements = await enhanceMetadata(llmConfig, entry, fileContent)
+      const updated = { ...entry }
+      if (improvements.description) updated.description = improvements.description
+      if (improvements.intents?.length) updated.intents = improvements.intents
+      if (improvements.tags?.length) updated.tags = improvements.tags
+      enhanced.push(updated)
+    } catch {
+      // LLM enhancement failed for this entry, keep original
+      enhanced.push(entry)
+    }
   }
+  return { entries: enhanced }
 }
 
 export function buildSearchText(entry: StashEntry): string {
@@ -190,22 +357,15 @@ export function buildSearchText(entry: StashEntry): string {
   if (entry.description) parts.push(entry.description)
   if (entry.tags) parts.push(entry.tags.join(" "))
   if (entry.examples) parts.push(entry.examples.join(" "))
+  if (entry.aliases) parts.push(entry.aliases.join(" "))
+  if (entry.intents) parts.push(entry.intents.join(" "))
   if (entry.intent) {
     if (entry.intent.when) parts.push(entry.intent.when)
     if (entry.intent.input) parts.push(entry.intent.input)
     if (entry.intent.output) parts.push(entry.intent.output)
   }
-  return parts.join(" ").toLowerCase()
-}
-
-function resolveStashDirForIndex(): string {
-  const raw = process.env.AGENTIKIT_STASH_DIR?.trim()
-  if (!raw) {
-    throw new Error("AGENTIKIT_STASH_DIR is not set. Run 'agentikit init' first.")
-  }
-  const stashDir = path.resolve(raw)
-  if (!fs.existsSync(stashDir) || !fs.statSync(stashDir).isDirectory()) {
-    throw new Error(`AGENTIKIT_STASH_DIR does not exist or is not a directory: "${stashDir}"`)
+  if (entry.toc) {
+    parts.push(entry.toc.map((h) => h.text).join(" "))
   }
-  return stashDir
+  return parts.join(" ").toLowerCase()
 }

package/src/init.ts

@@ -0,0 +1,106 @@
+/**
+ * Agentikit initialization logic.
+ *
+ * Creates the stash directory structure, sets the AGENTIKIT_STASH_DIR
+ * environment variable, and ensures ripgrep is available.
+ */
+
+import { spawnSync } from "node:child_process"
+import fs from "node:fs"
+import path from "node:path"
+import { IS_WINDOWS, TYPE_DIRS } from "./common"
+import { ensureRg } from "./ripgrep-install"
+import { getConfigPath, saveConfig, DEFAULT_CONFIG } from "./config"
+
+export interface InitResponse {
+  stashDir: string
+  created: boolean
+  envSet: boolean
+  profileUpdated?: string
+  configPath: string
+  ripgrep?: {
+    rgPath: string
+    installed: boolean
+    version: string
+  }
+}
+
+export function agentikitInit(): InitResponse {
+  let stashDir: string
+  if (IS_WINDOWS) {
+    const userProfile = process.env.USERPROFILE?.trim()
+    if (!userProfile) {
+      throw new Error("Unable to determine Documents folder. Ensure USERPROFILE is set.")
+    }
+    const docs = path.join(userProfile, "Documents")
+    stashDir = path.join(docs, "agentikit")
+  } else {
+    const home = process.env.HOME?.trim()
+    if (!home) {
+      throw new Error("Unable to determine home directory. Set HOME.")
+    }
+    stashDir = path.join(home, "agentikit")
+  }
+
+  let created = false
+  if (!fs.existsSync(stashDir)) {
+    fs.mkdirSync(stashDir, { recursive: true })
+    created = true
+  }
+
+  for (const sub of Object.values(TYPE_DIRS)) {
+    const subDir = path.join(stashDir, sub)
+    if (!fs.existsSync(subDir)) {
+      fs.mkdirSync(subDir, { recursive: true })
+    }
+  }
+
+  let envSet = false
+  let profileUpdated: string | undefined
+
+  if (IS_WINDOWS) {
+    const result = spawnSync("setx", ["AGENTIKIT_STASH_DIR", stashDir], {
+      encoding: "utf8",
+      timeout: 10_000,
+    })
+    envSet = result.status === 0
+  } else {
+    const shell = process.env.SHELL || ""
+    const homeDir = process.env.HOME! // already validated non-empty above
+    let profile: string
+    if (shell.endsWith("/zsh")) {
+      profile = path.join(homeDir, ".zshrc")
+    } else if (shell.endsWith("/bash")) {
+      profile = path.join(homeDir, ".bashrc")
+    } else {
+      profile = path.join(homeDir, ".profile")
+    }
+
+    const exportLine = `export AGENTIKIT_STASH_DIR="${stashDir}"`
+    const existing = fs.existsSync(profile) ? fs.readFileSync(profile, "utf8") : ""
+    if (!existing.includes("AGENTIKIT_STASH_DIR")) {
+      fs.appendFileSync(profile, `\n# Agentikit stash directory\n${exportLine}\n`)
+      envSet = true
+      profileUpdated = profile
+    }
+  }
+
+  // Create default config.json if it doesn't exist
+  const configPath = getConfigPath(stashDir)
+  if (!fs.existsSync(configPath)) {
+    saveConfig(DEFAULT_CONFIG, stashDir)
+  }
+
+  process.env.AGENTIKIT_STASH_DIR = stashDir
+
+  // Ensure ripgrep is available (install to stash/bin if needed)
+  let ripgrep: InitResponse["ripgrep"]
+  try {
+    const rgResult = ensureRg(stashDir)
+    ripgrep = rgResult
+  } catch {
+    // Non-fatal: ripgrep is optional, search works without it
+  }
+
+  return { stashDir, created, envSet, profileUpdated, configPath, ripgrep }
+}

package/src/llm.ts

@@ -0,0 +1,124 @@
+import type { LlmConnectionConfig } from "./config"
+import type { StashEntry } from "./metadata"
+
+// ── OpenAI-compatible chat completions ──────────────────────────────────────
+
+interface ChatMessage {
+  role: "system" | "user" | "assistant"
+  content: string
+}
+
+interface ChatCompletionResponse {
+  choices: Array<{ message: { content: string } }>
+}
+
+async function chatCompletion(
+  config: LlmConnectionConfig,
+  messages: ChatMessage[],
+): Promise<string> {
+  const headers: Record<string, string> = { "Content-Type": "application/json" }
+  if (config.apiKey) {
+    headers["Authorization"] = `Bearer ${config.apiKey}`
+  }
+
+  const response = await fetch(config.endpoint, {
+    method: "POST",
+    headers,
+    body: JSON.stringify({
+      model: config.model,
+      messages,
+      temperature: 0.3,
+      max_tokens: 512,
+    }),
+  })
+
+  if (!response.ok) {
+    const body = await response.text().catch(() => "")
+    throw new Error(`LLM request failed (${response.status}): ${body}`)
+  }
+
+  const json = (await response.json()) as ChatCompletionResponse
+  return json.choices?.[0]?.message?.content?.trim() ?? ""
+}
+
+// ── Metadata Enhancement ────────────────────────────────────────────────────
+
+const SYSTEM_PROMPT = `You are a metadata generator for a developer tool registry. Given a tool/skill/command/agent entry, generate improved metadata. Respond with ONLY valid JSON, no markdown fencing.`
+
+/**
+ * Use an LLM to enhance a stash entry's metadata: improve description,
+ * generate intents, and suggest tags.
+ */
+export async function enhanceMetadata(
+  config: LlmConnectionConfig,
+  entry: StashEntry,
+  fileContent?: string,
+): Promise<{ description?: string; intents?: string[]; tags?: string[] }> {
+  const contextParts = [
+    `Name: ${entry.name}`,
+    `Type: ${entry.type}`,
+  ]
+  if (entry.description) contextParts.push(`Current description: ${entry.description}`)
+  if (entry.tags?.length) contextParts.push(`Current tags: ${entry.tags.join(", ")}`)
+  if (fileContent) {
+    // Limit content to first 2000 chars to stay within token limits
+    const truncated = fileContent.length > 2000
+      ? fileContent.slice(0, 2000) + "\n... (truncated)"
+      : fileContent
+    contextParts.push(`File content:\n${truncated}`)
+  }
+
+  const userPrompt = `${contextParts.join("\n")}
+
+Generate improved metadata for this ${entry.type}. Return JSON with these fields:
+- "description": a clear, concise one-sentence description of what this does
+- "intents": an array of 3-6 natural language task phrases an agent might use to find this (e.g. "deploy a docker container", "run database migrations")
+- "tags": an array of 3-8 relevant keyword tags
+
+Return ONLY the JSON object, no explanation.`
+
+  const raw = await chatCompletion(config, [
+    { role: "system", content: SYSTEM_PROMPT },
+    { role: "user", content: userPrompt },
+  ])
+
+  try {
+    // Strip markdown code fences if present
+    const cleaned = raw.replace(/^```(?:json)?\s*\n?/i, "").replace(/\n?```\s*$/i, "")
+    const parsed = JSON.parse(cleaned) as Record<string, unknown>
+    const result: { description?: string; intents?: string[]; tags?: string[] } = {}
+
+    if (typeof parsed.description === "string" && parsed.description) {
+      result.description = parsed.description
+    }
+    if (Array.isArray(parsed.intents)) {
+      result.intents = parsed.intents.filter(
+        (s): s is string => typeof s === "string" && s.trim().length > 0,
+      ).slice(0, 8)
+    }
+    if (Array.isArray(parsed.tags)) {
+      result.tags = parsed.tags.filter(
+        (s): s is string => typeof s === "string" && s.trim().length > 0,
+      ).slice(0, 10)
+    }
+
+    return result
+  } catch {
+    // LLM returned unparseable output, return empty
+    return {}
+  }
+}
+
+/**
+ * Check if the LLM endpoint is reachable.
+ */
+export async function isLlmAvailable(config: LlmConnectionConfig): Promise<boolean> {
+  try {
+    const result = await chatCompletion(config, [
+      { role: "user", content: "Respond with just the word: ok" },
+    ])
+    return result.length > 0
+  } catch {
+    return false
+  }
+}