agentikit 0.0.8 → 0.0.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/README.md +135 -117
  2. package/dist/index.d.ts +13 -3
  3. package/dist/index.js +7 -1
  4. package/dist/src/asset-spec.d.ts +2 -0
  5. package/dist/src/asset-spec.js +22 -3
  6. package/dist/src/asset-type-handler.d.ts +27 -0
  7. package/dist/src/asset-type-handler.js +33 -0
  8. package/dist/src/cli.js +335 -100
  9. package/dist/src/common.d.ts +6 -1
  10. package/dist/src/common.js +18 -4
  11. package/dist/src/config-cli.d.ts +9 -0
  12. package/dist/src/config-cli.js +473 -0
  13. package/dist/src/config.d.ts +25 -6
  14. package/dist/src/config.js +188 -28
  15. package/dist/src/db.d.ts +46 -0
  16. package/dist/src/db.js +299 -0
  17. package/dist/src/embedder.js +12 -7
  18. package/dist/src/github.d.ts +4 -0
  19. package/dist/src/github.js +19 -0
  20. package/dist/src/handlers/agent-handler.d.ts +2 -0
  21. package/dist/src/handlers/agent-handler.js +26 -0
  22. package/dist/src/handlers/command-handler.d.ts +2 -0
  23. package/dist/src/handlers/command-handler.js +23 -0
  24. package/dist/src/handlers/index.d.ts +6 -0
  25. package/dist/src/handlers/index.js +23 -0
  26. package/dist/src/handlers/knowledge-handler.d.ts +2 -0
  27. package/dist/src/handlers/knowledge-handler.js +56 -0
  28. package/dist/src/handlers/markdown-helpers.d.ts +7 -0
  29. package/dist/src/handlers/markdown-helpers.js +15 -0
  30. package/dist/src/handlers/script-handler.d.ts +2 -0
  31. package/dist/src/handlers/script-handler.js +78 -0
  32. package/dist/src/handlers/skill-handler.d.ts +2 -0
  33. package/dist/src/handlers/skill-handler.js +30 -0
  34. package/dist/src/handlers/tool-handler.d.ts +2 -0
  35. package/dist/src/handlers/tool-handler.js +58 -0
  36. package/dist/src/indexer.d.ts +1 -23
  37. package/dist/src/indexer.js +162 -155
  38. package/dist/src/init.d.ts +2 -2
  39. package/dist/src/init.js +21 -9
  40. package/dist/src/llm.js +4 -3
  41. package/dist/src/metadata.d.ts +1 -1
  42. package/dist/src/metadata.js +22 -64
  43. package/dist/src/origin-resolve.d.ts +19 -0
  44. package/dist/src/origin-resolve.js +53 -0
  45. package/dist/src/registry-install.d.ts +11 -0
  46. package/dist/src/registry-install.js +315 -0
  47. package/dist/src/registry-resolve.d.ts +3 -0
  48. package/dist/src/registry-resolve.js +299 -0
  49. package/dist/src/registry-search.d.ts +27 -0
  50. package/dist/src/registry-search.js +263 -0
  51. package/dist/src/registry-types.d.ts +62 -0
  52. package/dist/src/registry-types.js +1 -0
  53. package/dist/src/stash-add.d.ts +4 -0
  54. package/dist/src/stash-add.js +59 -0
  55. package/dist/src/stash-clone.d.ts +22 -0
  56. package/dist/src/stash-clone.js +83 -0
  57. package/dist/src/stash-ref.d.ts +27 -3
  58. package/dist/src/stash-ref.js +63 -24
  59. package/dist/src/stash-registry.d.ts +18 -0
  60. package/dist/src/stash-registry.js +221 -0
  61. package/dist/src/stash-resolve.js +3 -0
  62. package/dist/src/stash-search.d.ts +3 -1
  63. package/dist/src/stash-search.js +357 -138
  64. package/dist/src/stash-show.d.ts +1 -1
  65. package/dist/src/stash-show.js +28 -89
  66. package/dist/src/stash-source.d.ts +24 -0
  67. package/dist/src/stash-source.js +81 -0
  68. package/dist/src/stash-types.d.ts +175 -1
  69. package/dist/src/stash.d.ts +9 -1
  70. package/dist/src/stash.js +5 -0
  71. package/dist/src/tool-runner.d.ts +1 -1
  72. package/dist/src/tool-runner.js +18 -5
  73. package/package.json +7 -2
  74. package/src/asset-spec.ts +20 -4
  75. package/src/asset-type-handler.ts +77 -0
  76. package/src/cli.ts +354 -103
  77. package/src/common.ts +23 -5
  78. package/src/config-cli.ts +499 -0
  79. package/src/config.ts +218 -37
  80. package/src/db.ts +411 -0
  81. package/src/embedder.ts +22 -11
  82. package/src/github.ts +21 -0
  83. package/src/handlers/agent-handler.ts +32 -0
  84. package/src/handlers/command-handler.ts +29 -0
  85. package/src/handlers/index.ts +25 -0
  86. package/src/handlers/knowledge-handler.ts +62 -0
  87. package/src/handlers/markdown-helpers.ts +19 -0
  88. package/src/handlers/script-handler.ts +92 -0
  89. package/src/handlers/skill-handler.ts +37 -0
  90. package/src/handlers/tool-handler.ts +71 -0
  91. package/src/indexer.ts +208 -187
  92. package/src/init.ts +17 -9
  93. package/src/llm.ts +4 -3
  94. package/src/metadata.ts +21 -65
  95. package/src/origin-resolve.ts +67 -0
  96. package/src/registry-install.ts +361 -0
  97. package/src/registry-resolve.ts +341 -0
  98. package/src/registry-search.ts +335 -0
  99. package/src/registry-types.ts +72 -0
  100. package/src/stash-add.ts +63 -0
  101. package/src/stash-clone.ts +127 -0
  102. package/src/stash-ref.ts +84 -26
  103. package/src/stash-registry.ts +259 -0
  104. package/src/stash-resolve.ts +3 -0
  105. package/src/stash-search.ts +425 -155
  106. package/src/stash-show.ts +33 -82
  107. package/src/stash-source.ts +103 -0
  108. package/src/stash-types.ts +186 -1
  109. package/src/stash.ts +23 -0
  110. package/src/tool-runner.ts +18 -5
  111. package/dist/src/similarity.d.ts +0 -34
  112. package/src/similarity.ts +0 -271
@@ -0,0 +1,92 @@
1
+ import path from "node:path"
2
+ import { SCRIPT_EXTENSIONS, SCRIPT_EXTENSIONS_BROAD } from "../asset-spec"
3
+ import { hasErrnoCode, toPosix } from "../common"
4
+ import { buildToolInfo } from "../tool-runner"
5
+ import { extractDescriptionFromComments } from "../metadata"
6
+ import type { AssetTypeHandler, ShowInput } from "../asset-type-handler"
7
+ import type { ShowResponse, LocalSearchHit } from "../stash-types"
8
+ import type { StashEntry } from "../metadata"
9
+
10
+ /** Extensions that buildToolInfo can handle (tool-runner supported) */
11
+ const RUNNABLE_EXTENSIONS = SCRIPT_EXTENSIONS
12
+
13
+ export const scriptHandler: AssetTypeHandler = {
14
+ typeName: "script",
15
+ stashDir: "scripts",
16
+
17
+ isRelevantFile(fileName: string): boolean {
18
+ return SCRIPT_EXTENSIONS_BROAD.has(path.extname(fileName).toLowerCase())
19
+ },
20
+
21
+ toCanonicalName(typeRoot: string, filePath: string): string | undefined {
22
+ return toPosix(path.relative(typeRoot, filePath))
23
+ },
24
+
25
+ toAssetPath(typeRoot: string, name: string): string {
26
+ return path.join(typeRoot, name)
27
+ },
28
+
29
+ buildShowResponse(input: ShowInput): ShowResponse {
30
+ const ext = path.extname(input.path).toLowerCase()
31
+
32
+ // For extensions supported by tool-runner, show runCmd
33
+ if (RUNNABLE_EXTENSIONS.has(ext)) {
34
+ const stashDirs = input.stashDirs ?? []
35
+ const assetStashDir = stashDirs.find((d) =>
36
+ path.resolve(input.path).startsWith(path.resolve(d) + path.sep),
37
+ ) ?? stashDirs[0]
38
+
39
+ if (assetStashDir) {
40
+ try {
41
+ const toolInfo = buildToolInfo(assetStashDir, input.path)
42
+ return {
43
+ type: "script",
44
+ name: input.name,
45
+ path: input.path,
46
+ runCmd: toolInfo.runCmd,
47
+ kind: toolInfo.kind,
48
+ }
49
+ } catch {
50
+ // Fall through to content display
51
+ }
52
+ }
53
+ }
54
+
55
+ // For other extensions or when buildToolInfo fails, show file content
56
+ return {
57
+ type: "script",
58
+ name: input.name,
59
+ path: input.path,
60
+ content: input.content,
61
+ }
62
+ },
63
+
64
+ enrichSearchHit(hit: LocalSearchHit, stashDir: string): void {
65
+ const ext = path.extname(hit.path).toLowerCase()
66
+ if (!RUNNABLE_EXTENSIONS.has(ext)) return
67
+
68
+ try {
69
+ const toolInfo = buildToolInfo(stashDir, hit.path)
70
+ hit.runCmd = toolInfo.runCmd
71
+ hit.kind = toolInfo.kind
72
+ } catch (error: unknown) {
73
+ if (!hasErrnoCode(error, "ENOENT")) throw error
74
+ }
75
+ },
76
+
77
+ defaultUsageGuide: [
78
+ "Use the hit's runCmd for execution when available, or run the script directly with the appropriate interpreter.",
79
+ "Use `akm show <openRef>` to inspect the script before running it.",
80
+ ],
81
+
82
+ extractTypeMetadata(entry: StashEntry, file: string, ext: string): void {
83
+ if (ext !== ".md") {
84
+ const commentDesc = extractDescriptionFromComments(file)
85
+ if (commentDesc && !entry.description) {
86
+ entry.description = commentDesc
87
+ entry.source = "comments"
88
+ entry.confidence = 0.7
89
+ }
90
+ }
91
+ },
92
+ }
@@ -0,0 +1,37 @@
1
+ import path from "node:path"
2
+ import { toPosix } from "../common"
3
+ import type { AssetTypeHandler, ShowInput } from "../asset-type-handler"
4
+ import type { ShowResponse } from "../stash-types"
5
+
6
+ export const skillHandler: AssetTypeHandler = {
7
+ typeName: "skill",
8
+ stashDir: "skills",
9
+
10
+ isRelevantFile(fileName: string): boolean {
11
+ return fileName === "SKILL.md"
12
+ },
13
+
14
+ toCanonicalName(typeRoot: string, filePath: string): string | undefined {
15
+ const relDir = toPosix(path.dirname(path.relative(typeRoot, filePath)))
16
+ if (!relDir || relDir === ".") return undefined
17
+ return relDir
18
+ },
19
+
20
+ toAssetPath(typeRoot: string, name: string): string {
21
+ return path.join(typeRoot, name, "SKILL.md")
22
+ },
23
+
24
+ buildShowResponse(input: ShowInput): ShowResponse {
25
+ return {
26
+ type: "skill",
27
+ name: input.name,
28
+ path: input.path,
29
+ content: input.content,
30
+ }
31
+ },
32
+
33
+ defaultUsageGuide: [
34
+ "Read and apply the skill instructions as written, then adapt examples to your current repo state and task.",
35
+ "Use `akm show <openRef>` to read the full SKILL.md for required steps and constraints.",
36
+ ],
37
+ }
@@ -0,0 +1,71 @@
1
+ import path from "node:path"
2
+ import { SCRIPT_EXTENSIONS } from "../asset-spec"
3
+ import { hasErrnoCode, toPosix } from "../common"
4
+ import { buildToolInfo } from "../tool-runner"
5
+ import { extractDescriptionFromComments } from "../metadata"
6
+ import type { AssetTypeHandler, ShowInput } from "../asset-type-handler"
7
+ import type { ShowResponse, LocalSearchHit } from "../stash-types"
8
+ import type { StashEntry } from "../metadata"
9
+
10
+ export const toolHandler: AssetTypeHandler = {
11
+ typeName: "tool",
12
+ stashDir: "tools",
13
+
14
+ isRelevantFile(fileName: string): boolean {
15
+ return SCRIPT_EXTENSIONS.has(path.extname(fileName).toLowerCase())
16
+ },
17
+
18
+ toCanonicalName(typeRoot: string, filePath: string): string | undefined {
19
+ return toPosix(path.relative(typeRoot, filePath))
20
+ },
21
+
22
+ toAssetPath(typeRoot: string, name: string): string {
23
+ return path.join(typeRoot, name)
24
+ },
25
+
26
+ buildShowResponse(input: ShowInput): ShowResponse {
27
+ const stashDirs = input.stashDirs ?? []
28
+ const assetStashDir = stashDirs.find((d) =>
29
+ path.resolve(input.path).startsWith(path.resolve(d) + path.sep),
30
+ ) ?? stashDirs[0]
31
+
32
+ if (!assetStashDir) {
33
+ return { type: "tool", name: input.name, path: input.path, content: input.content }
34
+ }
35
+
36
+ const toolInfo = buildToolInfo(assetStashDir, input.path)
37
+ return {
38
+ type: "tool",
39
+ name: input.name,
40
+ path: input.path,
41
+ runCmd: toolInfo.runCmd,
42
+ kind: toolInfo.kind,
43
+ }
44
+ },
45
+
46
+ enrichSearchHit(hit: LocalSearchHit, stashDir: string): void {
47
+ try {
48
+ const toolInfo = buildToolInfo(stashDir, hit.path)
49
+ hit.runCmd = toolInfo.runCmd
50
+ hit.kind = toolInfo.kind
51
+ } catch (error: unknown) {
52
+ if (!hasErrnoCode(error, "ENOENT")) throw error
53
+ }
54
+ },
55
+
56
+ defaultUsageGuide: [
57
+ "Use the hit's runCmd for execution so runtime and working directory stay correct.",
58
+ "Use `akm show <openRef>` to inspect the tool before running it.",
59
+ ],
60
+
61
+ extractTypeMetadata(entry: StashEntry, file: string, ext: string): void {
62
+ if (SCRIPT_EXTENSIONS.has(ext) && ext !== ".md") {
63
+ const commentDesc = extractDescriptionFromComments(file)
64
+ if (commentDesc && !entry.description) {
65
+ entry.description = commentDesc
66
+ entry.source = "comments"
67
+ entry.confidence = 0.7
68
+ }
69
+ }
70
+ },
71
+ }
package/src/indexer.ts CHANGED
@@ -9,33 +9,27 @@ import {
9
9
  writeStashFile,
10
10
  generateMetadata,
11
11
  } from "./metadata"
12
- import { TfIdfAdapter, type ScoredEntry, type SerializedTfIdf } from "./similarity"
13
12
  import { walkStash } from "./walker"
14
- import type { EmbeddingVector } from "./embedder"
15
13
  import type { LlmConnectionConfig } from "./config"
14
+ import {
15
+ openDatabase,
16
+ closeDatabase,
17
+ getDbPath,
18
+ getMeta,
19
+ setMeta,
20
+ upsertEntry,
21
+ deleteEntriesByDir,
22
+ rebuildFts,
23
+ upsertEmbedding,
24
+ getEntriesByDir,
25
+ getEntryCount,
26
+ isVecAvailable,
27
+ DB_VERSION,
28
+ type DbIndexedEntry,
29
+ } from "./db"
16
30
 
17
31
  // ── Types ───────────────────────────────────────────────────────────────────
18
32
 
19
- export interface IndexedEntry {
20
- entry: StashEntry
21
- path: string
22
- dirPath: string
23
- embedding?: EmbeddingVector
24
- }
25
-
26
- export interface SearchIndex {
27
- version: number
28
- builtAt: string
29
- stashDir: string
30
- /** All stash directories that were indexed (primary + additional) */
31
- stashDirs?: string[]
32
- entries: IndexedEntry[]
33
- /** Serialized TF-IDF state (term frequencies, idf values) */
34
- tfidf?: SerializedTfIdf
35
- /** Whether embeddings are included in entries */
36
- hasEmbeddings?: boolean
37
- }
38
-
39
33
  export interface IndexResponse {
40
34
  stashDir: string
41
35
  totalEntries: number
@@ -45,211 +39,240 @@ export interface IndexResponse {
45
39
  directoriesScanned: number
46
40
  directoriesSkipped: number
47
41
  /** Timing counters in milliseconds */
48
- timing?: { totalMs: number; walkMs: number; embedMs: number; tfidfMs: number }
42
+ timing?: { totalMs: number; walkMs: number; embedMs: number; ftsMs: number }
49
43
  }
50
44
 
51
- // ── Constants ───────────────────────────────────────────────────────────────
45
+ // ── Indexer ──────────────────────────────────────────────────────────────────
52
46
 
53
- const INDEX_VERSION = 4
47
+ export async function agentikitIndex(options?: { stashDir?: string; full?: boolean }): Promise<IndexResponse> {
48
+ const stashDir = options?.stashDir || resolveStashDir()
54
49
 
55
- // ── Index Path ──────────────────────────────────────────────────────────────
50
+ // Load config and resolve all stash sources
51
+ const { loadConfig } = await import("./config.js")
52
+ const config = loadConfig()
53
+ const { resolveAllStashDirs } = await import("./stash-source.js")
54
+ const allStashDirs = resolveAllStashDirs(stashDir)
56
55
 
57
- export function getIndexPath(): string {
58
- const cacheDir = process.env.XDG_CACHE_HOME
59
- || path.join(process.env.HOME || process.env.USERPROFILE || "", ".cache")
60
- return path.join(cacheDir, "agentikit", "index.json")
61
- }
56
+ const t0 = Date.now()
57
+
58
+ // Open database pass embedding dimension from config if available
59
+ const dbPath = getDbPath()
60
+ const embeddingDim = config.embedding?.dimension
61
+ const db = openDatabase(dbPath, embeddingDim ? { embeddingDim } : undefined)
62
62
 
63
- export function loadSearchIndex(): SearchIndex | null {
64
- const indexPath = getIndexPath()
65
- if (!fs.existsSync(indexPath)) return null
66
63
  try {
67
- const raw = JSON.parse(fs.readFileSync(indexPath, "utf8"))
68
- if (raw?.version !== INDEX_VERSION) return null
69
- return raw as SearchIndex
70
- } catch {
71
- return null
72
- }
73
- }
64
+ // Check if we should do incremental
65
+ const prevStashDir = getMeta(db, "stashDir")
66
+ const prevBuiltAt = getMeta(db, "builtAt")
67
+ const isIncremental = !options?.full && prevStashDir === stashDir && !!prevBuiltAt
68
+ const builtAtMs = isIncremental ? new Date(prevBuiltAt!).getTime() : 0
69
+
70
+ if (options?.full || !isIncremental) {
71
+ // Wipe all entries for full rebuild or stashDir change
72
+ db.exec("DELETE FROM entries")
73
+ db.exec("DELETE FROM entries_fts")
74
+ if (isVecAvailable()) {
75
+ try { db.exec("DELETE FROM entries_vec") } catch { /* ignore */ }
76
+ }
77
+ }
74
78
 
75
- // ── Indexer ──────────────────────────────────────────────────────────────────
79
+ const tWalkStart = Date.now()
76
80
 
77
- export async function agentikitIndex(options?: { stashDir?: string; full?: boolean }): Promise<IndexResponse> {
78
- const stashDir = options?.stashDir || resolveStashDir()
81
+ // Walk stash dirs and index entries
82
+ const { scannedDirs, skippedDirs, generatedCount, dirsNeedingLlm } = indexEntries(db, allStashDirs, stashDir, isIncremental, builtAtMs)
79
83
 
80
- // Load config to get additional stash dirs and semantic search setting
81
- const { loadConfig } = await import("./config.js")
82
- const config = loadConfig(stashDir)
84
+ // Enhance entries with LLM if configured
85
+ await enhanceDirsWithLlm(db, config, dirsNeedingLlm)
83
86
 
84
- const allStashDirs = [stashDir]
85
- for (const d of config.additionalStashDirs) {
86
- try {
87
- if (fs.statSync(d).isDirectory() && !allStashDirs.includes(path.resolve(d))) {
88
- allStashDirs.push(path.resolve(d))
89
- }
90
- } catch { /* skip nonexistent dirs */ }
91
- }
87
+ const tWalkEnd = Date.now()
92
88
 
93
- const t0 = Date.now()
94
- const allEntries: IndexedEntry[] = []
95
- let generatedCount = 0
96
- let scannedDirs = 0
97
- let skippedDirs = 0
89
+ // Rebuild FTS after all inserts
90
+ rebuildFts(db)
91
+ const tFtsEnd = Date.now()
92
+
93
+ // Generate embeddings if semantic search is enabled
94
+ const hasEmbeddings = await generateEmbeddingsForDb(db, config)
95
+
96
+ const tEmbedEnd = Date.now()
97
+
98
+ // Update metadata
99
+ setMeta(db, "version", String(DB_VERSION))
100
+ setMeta(db, "builtAt", new Date().toISOString())
101
+ setMeta(db, "stashDir", stashDir)
102
+ setMeta(db, "stashDirs", JSON.stringify(allStashDirs))
103
+ setMeta(db, "hasEmbeddings", hasEmbeddings ? "1" : "0")
104
+
105
+ const totalEntries = getEntryCount(db)
98
106
 
99
- // Load previous index for incremental mode
100
- const previousIndex = !options?.full ? loadSearchIndex() : null
101
- const isIncremental = previousIndex !== null && previousIndex.stashDir === stashDir
102
- const builtAtMs = isIncremental ? new Date(previousIndex.builtAt).getTime() : 0
103
-
104
- // Build lookup of previous entries by dirPath
105
- const previousEntriesByDir = new Map<string, IndexedEntry[]>()
106
- if (isIncremental) {
107
- for (const ie of previousIndex.entries) {
108
- const list = previousEntriesByDir.get(ie.dirPath) || []
109
- list.push(ie)
110
- previousEntriesByDir.set(ie.dirPath, list)
107
+ const tEnd = Date.now()
108
+
109
+ return {
110
+ stashDir,
111
+ totalEntries,
112
+ generatedMetadata: generatedCount,
113
+ indexPath: dbPath,
114
+ mode: isIncremental ? "incremental" : "full",
115
+ directoriesScanned: scannedDirs,
116
+ directoriesSkipped: skippedDirs,
117
+ timing: {
118
+ totalMs: tEnd - t0,
119
+ walkMs: tWalkEnd - tWalkStart,
120
+ embedMs: tEmbedEnd - tFtsEnd,
121
+ ftsMs: tFtsEnd - tWalkEnd,
122
+ },
111
123
  }
124
+ } finally {
125
+ closeDatabase(db)
112
126
  }
127
+ }
128
+
129
+ // ── Extracted helpers for agentikitIndex ─────────────────────────────────────
113
130
 
131
+ function indexEntries(
132
+ db: import("bun:sqlite").Database,
133
+ allStashDirs: string[],
134
+ stashDir: string,
135
+ isIncremental: boolean,
136
+ builtAtMs: number,
137
+ ): { scannedDirs: number; skippedDirs: number; generatedCount: number; dirsNeedingLlm: Array<{ dirPath: string; files: string[]; assetType: AgentikitAssetType; currentStashDir: string }> } {
138
+ let scannedDirs = 0
139
+ let skippedDirs = 0
140
+ let generatedCount = 0
114
141
  const seenPaths = new Set<string>()
115
- const tWalkStart = Date.now()
116
-
117
- for (const currentStashDir of allStashDirs) {
118
- for (const assetType of ASSET_TYPES as AgentikitAssetType[]) {
119
- const typeRoot = path.join(currentStashDir, TYPE_DIRS[assetType])
120
- try {
121
- if (!fs.statSync(typeRoot).isDirectory()) continue
122
- } catch { continue }
123
-
124
- // Group files by their immediate parent directory
125
- const dirGroups = walkStash(typeRoot, assetType)
126
-
127
- for (const { dirPath, files } of dirGroups) {
128
- // Deduplicate by dirPath across stash dirs
129
- if (seenPaths.has(path.resolve(dirPath))) continue
130
- seenPaths.add(path.resolve(dirPath))
131
-
132
- // Incremental: skip directories that haven't changed
133
- const prevEntries = previousEntriesByDir.get(dirPath)
134
- if (isIncremental && prevEntries && !isDirStale(dirPath, files, prevEntries, builtAtMs)) {
135
- allEntries.push(...prevEntries)
136
- skippedDirs++
137
- continue
138
- }
142
+ const dirsNeedingLlm: Array<{ dirPath: string; files: string[]; assetType: AgentikitAssetType; currentStashDir: string }> = []
139
143
 
140
- scannedDirs++
144
+ const insertTransaction = db.transaction(() => {
145
+ for (const currentStashDir of allStashDirs) {
146
+ for (const assetType of ASSET_TYPES as AgentikitAssetType[]) {
147
+ const typeRoot = path.join(currentStashDir, TYPE_DIRS[assetType])
148
+ try {
149
+ if (!fs.statSync(typeRoot).isDirectory()) continue
150
+ } catch { continue }
151
+
152
+ const dirGroups = walkStash(typeRoot, assetType)
153
+
154
+ for (const { dirPath, files } of dirGroups) {
155
+ if (seenPaths.has(path.resolve(dirPath))) continue
156
+ seenPaths.add(path.resolve(dirPath))
157
+
158
+ // Incremental: skip directories that haven't changed
159
+ if (isIncremental) {
160
+ const prevEntries = getEntriesByDir(db, dirPath)
161
+ if (prevEntries.length > 0 && !isDirStale(dirPath, files, prevEntries, builtAtMs)) {
162
+ skippedDirs++
163
+ continue
164
+ }
165
+ }
141
166
 
142
- // Try loading existing .stash.json
143
- let stash = loadStashFile(dirPath)
167
+ scannedDirs++
144
168
 
145
- if (stash) {
146
- const migration = migrateGeneratedSkillMetadata(stash, files, typeRoot)
147
- if (migration.changed) {
148
- stash = migration.stash
149
- writeStashFile(dirPath, stash)
150
- }
151
- }
169
+ // Delete old entries for this dir (will be re-inserted)
170
+ deleteEntriesByDir(db, dirPath)
152
171
 
153
- if (!stash) {
154
- // Generate metadata
155
- stash = generateMetadata(dirPath, assetType, files, typeRoot)
156
- // Enhance with LLM if configured
157
- if (config.llm && stash.entries.length > 0) {
158
- stash = await enhanceStashWithLlm(config.llm, stash, dirPath, files)
159
- }
160
- if (stash.entries.length > 0) {
161
- writeStashFile(dirPath, stash)
162
- generatedCount += stash.entries.length
172
+ // Try loading existing .stash.json (user metadata overrides)
173
+ let stash = loadStashFile(dirPath)
174
+
175
+ if (stash) {
176
+ const migration = migrateGeneratedSkillMetadata(stash, files, typeRoot)
177
+ if (migration.changed) {
178
+ stash = migration.stash
179
+ writeStashFile(dirPath, stash)
180
+ }
163
181
  }
164
- }
165
182
 
166
- if (stash) {
167
- for (const entry of stash.entries) {
168
- const entryPath = entry.entry
169
- ? path.join(dirPath, entry.entry)
170
- : files[0] || dirPath
171
- allEntries.push({ entry, path: entryPath, dirPath })
183
+ if (!stash) {
184
+ // Generate metadata heuristically
185
+ stash = generateMetadata(dirPath, assetType, files, typeRoot)
186
+ if (stash.entries.length > 0) {
187
+ writeStashFile(dirPath, stash)
188
+ generatedCount += stash.entries.length
189
+ }
172
190
  }
173
- }
174
- }
175
- }
176
- }
177
191
 
178
- const tWalkEnd = Date.now()
179
-
180
- // Build TF-IDF index
181
- const adapter = new TfIdfAdapter()
182
- const scoredEntries: ScoredEntry[] = allEntries.map((ie) => ({
183
- id: `${ie.entry.type}:${ie.entry.name}`,
184
- text: buildSearchText(ie.entry),
185
- entry: ie.entry,
186
- path: ie.path,
187
- }))
188
- adapter.buildIndex(scoredEntries)
189
- const tTfidfEnd = Date.now()
190
-
191
- // Generate embeddings if semantic search is enabled
192
- let hasEmbeddings = false
193
- if (config.semanticSearch) {
194
- try {
195
- const { embed } = await import("./embedder.js")
196
- for (const ie of allEntries) {
197
- if (!ie.embedding) {
198
- const text = buildSearchText(ie.entry)
199
- ie.embedding = await embed(text, config.embedding)
192
+ if (stash) {
193
+ for (const entry of stash.entries) {
194
+ const entryPath = entry.entry
195
+ ? path.join(dirPath, entry.entry)
196
+ : files[0] || dirPath
197
+ const entryKey = `${currentStashDir}:${entry.type}:${entry.name}`
198
+ const searchText = buildSearchText(entry)
199
+
200
+ upsertEntry(db, entryKey, dirPath, entryPath, currentStashDir, entry, searchText)
201
+ }
202
+
203
+ // Collect dirs needing LLM enhancement during the first walk
204
+ if (stash.entries.some((e) => e.generated)) {
205
+ dirsNeedingLlm.push({ dirPath, files, assetType, currentStashDir })
206
+ }
207
+ }
200
208
  }
201
209
  }
202
- hasEmbeddings = true
203
- } catch {
204
- // Embedding provider not available, continue without embeddings
205
210
  }
206
- }
211
+ })
207
212
 
208
- const tEmbedEnd = Date.now()
213
+ insertTransaction()
209
214
 
210
- // Persist index
211
- const indexPath = getIndexPath()
212
- const indexDir = path.dirname(indexPath)
213
- if (!fs.existsSync(indexDir)) {
214
- fs.mkdirSync(indexDir, { recursive: true })
215
- }
215
+ return { scannedDirs, skippedDirs, generatedCount, dirsNeedingLlm }
216
+ }
216
217
 
217
- const index: SearchIndex = {
218
- version: INDEX_VERSION,
219
- builtAt: new Date().toISOString(),
220
- stashDir,
221
- stashDirs: allStashDirs,
222
- entries: allEntries,
223
- tfidf: adapter.serialize(),
224
- hasEmbeddings,
218
+ async function enhanceDirsWithLlm(
219
+ db: import("bun:sqlite").Database,
220
+ config: import("./config").AgentikitConfig,
221
+ dirsNeedingLlm: Array<{ dirPath: string; files: string[]; assetType: AgentikitAssetType; currentStashDir: string }>,
222
+ ): Promise<void> {
223
+ if (!config.llm || dirsNeedingLlm.length === 0) return
224
+
225
+ for (const { dirPath, files, currentStashDir } of dirsNeedingLlm) {
226
+ let stash = loadStashFile(dirPath)
227
+ if (!stash) continue
228
+ stash = await enhanceStashWithLlm(config.llm, stash, dirPath, files)
229
+ writeStashFile(dirPath, stash)
230
+
231
+ // Re-upsert enhanced entries
232
+ for (const entry of stash.entries) {
233
+ const entryPath = entry.entry ? path.join(dirPath, entry.entry) : files[0] || dirPath
234
+ const entryKey = `${currentStashDir}:${entry.type}:${entry.name}`
235
+ const searchText = buildSearchText(entry)
236
+ upsertEntry(db, entryKey, dirPath, entryPath, currentStashDir, entry, searchText)
237
+ }
225
238
  }
226
- fs.writeFileSync(indexPath, JSON.stringify(index) + "\n", "utf8")
239
+ }
227
240
 
228
- const tEnd = Date.now()
241
+ async function generateEmbeddingsForDb(
242
+ db: import("bun:sqlite").Database,
243
+ config: import("./config").AgentikitConfig,
244
+ ): Promise<boolean> {
245
+ if (!config.semanticSearch || !isVecAvailable()) return false
229
246
 
230
- return {
231
- stashDir,
232
- totalEntries: allEntries.length,
233
- generatedMetadata: generatedCount,
234
- indexPath,
235
- mode: isIncremental ? "incremental" : "full",
236
- directoriesScanned: scannedDirs,
237
- directoriesSkipped: skippedDirs,
238
- timing: {
239
- totalMs: tEnd - t0,
240
- walkMs: tWalkEnd - tWalkStart, // includes metadata generation (interleaved)
241
- embedMs: tEmbedEnd - tTfidfEnd,
242
- tfidfMs: tTfidfEnd - tWalkEnd,
243
- },
247
+ try {
248
+ const { embed } = await import("./embedder.js")
249
+ const allEntries = getAllEntriesForEmbedding(db)
250
+ for (const { id, searchText } of allEntries) {
251
+ const embedding = await embed(searchText, config.embedding)
252
+ upsertEmbedding(db, id, embedding)
253
+ }
254
+ return true
255
+ } catch (error) {
256
+ console.warn("Embedding generation failed, continuing without:", error instanceof Error ? error.message : String(error))
257
+ return false
244
258
  }
245
259
  }
246
260
 
247
261
  // ── Helpers ─────────────────────────────────────────────────────────────────
248
262
 
263
+ function getAllEntriesForEmbedding(db: import("bun:sqlite").Database): Array<{ id: number; searchText: string }> {
264
+ return db
265
+ .prepare(`
266
+ SELECT e.id, e.search_text AS searchText FROM entries e
267
+ WHERE NOT EXISTS (SELECT 1 FROM entries_vec v WHERE v.id = e.id)
268
+ `)
269
+ .all() as Array<{ id: number; searchText: string }>
270
+ }
271
+
249
272
  function isDirStale(
250
273
  dirPath: string,
251
274
  currentFiles: string[],
252
- previousEntries: IndexedEntry[],
275
+ previousEntries: DbIndexedEntry[],
253
276
  builtAtMs: number,
254
277
  ): boolean {
255
278
  // Check if file set changed (additions or deletions)
@@ -327,7 +350,6 @@ async function enhanceStashWithLlm(
327
350
  const enhanced: StashEntry[] = []
328
351
  for (const entry of stash.entries) {
329
352
  try {
330
- // Find the file matching this entry for content context
331
353
  const entryFile = entry.entry
332
354
  ? files.find((f) => path.basename(f) === entry.entry) ?? files[0]
333
355
  : files[0]
@@ -345,7 +367,6 @@ async function enhanceStashWithLlm(
345
367
  if (improvements.tags?.length) updated.tags = improvements.tags
346
368
  enhanced.push(updated)
347
369
  } catch {
348
- // LLM enhancement failed for this entry, keep original
349
370
  enhanced.push(entry)
350
371
  }
351
372
  }