@pi-unipi/unipi 2.6.0 → 2.6.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (205) hide show
  1. package/CHANGELOG.md +19 -0
  2. package/package.json +21 -22
  3. package/packages/ask-user/package.json +2 -2
  4. package/packages/ask-user/settings-tui.ts +18 -34
  5. package/packages/autocomplete/package.json +1 -1
  6. package/packages/autocomplete/src/constants.ts +2 -15
  7. package/packages/autocomplete/src/provider.ts +3 -17
  8. package/packages/btw/extensions/btw.ts +18 -45
  9. package/packages/btw/package.json +2 -2
  10. package/packages/compactor/README.md +2 -2
  11. package/packages/compactor/package.json +3 -3
  12. package/packages/compactor/skills/compactor/SKILL.md +1 -1
  13. package/packages/compactor/skills/compactor-detail/SKILL.md +3 -5
  14. package/packages/compactor/skills/compactor-doctor/SKILL.md +1 -1
  15. package/packages/compactor/skills/compactor-stats/SKILL.md +1 -1
  16. package/packages/compactor/src/commands/index.ts +47 -78
  17. package/packages/compactor/src/compaction/brief.ts +161 -90
  18. package/packages/compactor/src/compaction/build-sections.ts +3 -4
  19. package/packages/compactor/src/compaction/compact-args.ts +86 -0
  20. package/packages/compactor/src/compaction/cut.ts +270 -28
  21. package/packages/compactor/src/compaction/drill-down.ts +261 -0
  22. package/packages/compactor/src/compaction/format-recall.ts +96 -0
  23. package/packages/compactor/src/compaction/format.ts +8 -3
  24. package/packages/compactor/src/compaction/hooks.ts +248 -72
  25. package/packages/compactor/src/compaction/merge.ts +34 -4
  26. package/packages/compactor/src/compaction/rank.ts +270 -0
  27. package/packages/compactor/src/compaction/recall-scope.ts +28 -0
  28. package/packages/compactor/src/compaction/search-entries.ts +333 -96
  29. package/packages/compactor/src/compaction/skill-collapse.ts +35 -0
  30. package/packages/compactor/src/compaction/summarize.ts +37 -6
  31. package/packages/compactor/src/compaction/token-estimate.ts +104 -0
  32. package/packages/compactor/src/compaction/touched-files.ts +35 -0
  33. package/packages/compactor/src/config/manager.ts +2 -27
  34. package/packages/compactor/src/config/presets.ts +0 -2
  35. package/packages/compactor/src/config/schema.ts +2 -16
  36. package/packages/compactor/src/executor/executor.ts +6 -15
  37. package/packages/compactor/src/executor/runtime.ts +2 -12
  38. package/packages/compactor/src/index.ts +4 -121
  39. package/packages/compactor/src/info-screen.ts +3 -10
  40. package/packages/compactor/src/security/evaluator.ts +0 -53
  41. package/packages/compactor/src/security/policy.ts +7 -8
  42. package/packages/compactor/src/session/db.ts +0 -6
  43. package/packages/compactor/src/tools/ctx-execute-file.ts +0 -5
  44. package/packages/compactor/src/tools/register.ts +27 -50
  45. package/packages/compactor/src/tools/vcc-recall.ts +86 -48
  46. package/packages/compactor/src/tui/settings-overlay.ts +20 -40
  47. package/packages/compactor/src/types.ts +43 -100
  48. package/packages/core/constants.ts +1 -28
  49. package/packages/core/events.ts +1 -83
  50. package/packages/core/index.ts +1 -0
  51. package/packages/core/model-cache.ts +18 -12
  52. package/packages/core/package.json +3 -1
  53. package/packages/core/tui-overlay.ts +89 -0
  54. package/packages/core/tui-width.ts +18 -0
  55. package/packages/core/utils.ts +43 -0
  56. package/packages/footer/package.json +2 -2
  57. package/packages/footer/src/help.ts +2 -2
  58. package/packages/footer/src/index.ts +9 -10
  59. package/packages/footer/src/presets.ts +1 -27
  60. package/packages/footer/src/registry/index.ts +0 -23
  61. package/packages/footer/src/rendering/icons.ts +0 -13
  62. package/packages/footer/src/rendering/renderer.ts +5 -40
  63. package/packages/footer/src/rendering/separators.ts +0 -10
  64. package/packages/footer/src/rendering/theme.ts +1 -15
  65. package/packages/footer/src/segments/compactor.ts +7 -14
  66. package/packages/footer/src/segments/core.ts +15 -54
  67. package/packages/footer/src/segments/kanboard.ts +5 -5
  68. package/packages/footer/src/segments/mcp.ts +5 -5
  69. package/packages/footer/src/segments/memory.ts +4 -4
  70. package/packages/footer/src/segments/notify.ts +2 -2
  71. package/packages/footer/src/segments/ralph.ts +3 -3
  72. package/packages/footer/src/segments/status-ext.ts +1 -1
  73. package/packages/footer/src/segments/workflow.ts +3 -3
  74. package/packages/footer/src/tps-tracker.ts +0 -8
  75. package/packages/footer/src/tui/settings-tui.ts +14 -31
  76. package/packages/footer/src/types.ts +0 -15
  77. package/packages/image/package.json +2 -2
  78. package/packages/image/src/register-providers.ts +0 -11
  79. package/packages/image/src/tools.ts +0 -3
  80. package/packages/image/src/tui/model-selector.ts +36 -71
  81. package/packages/info-screen/config.ts +0 -16
  82. package/packages/info-screen/core-groups.ts +0 -10
  83. package/packages/info-screen/index.ts +22 -13
  84. package/packages/info-screen/package.json +2 -2
  85. package/packages/info-screen/registry.ts +1 -72
  86. package/packages/info-screen/tui/info-overlay.ts +85 -100
  87. package/packages/info-screen/usage-parser.ts +12 -47
  88. package/packages/input-shortcuts/package.json +2 -2
  89. package/packages/input-shortcuts/src/index.ts +3 -3
  90. package/packages/input-shortcuts/src/registers.ts +0 -8
  91. package/packages/input-shortcuts/src/settings-overlay.ts +4 -13
  92. package/packages/input-shortcuts/src/undo-redo.ts +0 -12
  93. package/packages/kanboard/package.json +2 -2
  94. package/packages/mcp/package.json +2 -2
  95. package/packages/mcp/src/bridge/client.ts +0 -15
  96. package/packages/mcp/src/config/manager.ts +0 -38
  97. package/packages/mcp/src/config/schema.ts +0 -27
  98. package/packages/mcp/src/config/sync.ts +0 -29
  99. package/packages/mcp/src/index.ts +1 -1
  100. package/packages/mcp/src/tui/add-overlay.ts +6 -6
  101. package/packages/mcp/src/types.ts +0 -6
  102. package/packages/memory/embedding.ts +0 -31
  103. package/packages/memory/mempalace.ts +0 -16
  104. package/packages/memory/package.json +4 -8
  105. package/packages/memory/storage.ts +56 -753
  106. package/packages/milestone/hooks.ts +1 -6
  107. package/packages/milestone/milestone.ts +2 -41
  108. package/packages/milestone/package.json +2 -2
  109. package/packages/notify/README.md +1 -0
  110. package/packages/notify/commands.ts +61 -3
  111. package/packages/notify/events.ts +0 -10
  112. package/packages/notify/package.json +2 -2
  113. package/packages/notify/settings.ts +0 -19
  114. package/packages/notify/tui/gotify-setup.ts +108 -143
  115. package/packages/notify/tui/ntfy-setup.ts +137 -172
  116. package/packages/notify/tui/recap-model-selector.ts +107 -101
  117. package/packages/notify/tui/settings-overlay.ts +94 -117
  118. package/packages/notify/tui/telegram-setup.ts +48 -79
  119. package/packages/notify/types.ts +0 -2
  120. package/packages/ralph/index.ts +11 -35
  121. package/packages/ralph/package.json +3 -3
  122. package/packages/ralph/ralph-loop.ts +3 -44
  123. package/packages/subagents/package.json +2 -1
  124. package/packages/subagents/src/agent-manager.ts +0 -51
  125. package/packages/subagents/src/agent-runner.ts +6 -80
  126. package/packages/subagents/src/conversation-viewer.ts +1 -19
  127. package/packages/subagents/src/core-compat.ts +8 -25
  128. package/packages/subagents/src/custom-agents.ts +0 -1
  129. package/packages/subagents/src/index.ts +3 -48
  130. package/packages/subagents/src/types.ts +0 -11
  131. package/packages/subagents/src/widget.ts +5 -5
  132. package/packages/unipi/bundled.js +4468 -7769
  133. package/packages/unipi/index.ts +6 -3
  134. package/packages/updater/package.json +2 -2
  135. package/packages/updater/src/changelog.ts +1 -16
  136. package/packages/updater/src/index.ts +0 -5
  137. package/packages/updater/src/installer.ts +4 -26
  138. package/packages/updater/src/markdown.ts +6 -152
  139. package/packages/updater/src/settings.ts +0 -15
  140. package/packages/updater/src/tui/changelog-overlay.ts +42 -234
  141. package/packages/updater/src/tui/list-detail-overlay.ts +260 -0
  142. package/packages/updater/src/tui/readme-overlay.ts +53 -210
  143. package/packages/updater/src/tui/settings-overlay.ts +13 -26
  144. package/packages/updater/src/version.ts +2 -31
  145. package/packages/utility/package.json +2 -4
  146. package/packages/utility/src/commands.ts +4 -5
  147. package/packages/utility/src/herdr-sync.ts +136 -0
  148. package/packages/utility/src/index.ts +11 -68
  149. package/packages/utility/src/lifecycle/cleanup.ts +2 -16
  150. package/packages/utility/src/{diff/settings.ts → settings.ts} +42 -80
  151. package/packages/utility/src/tools/env.ts +0 -0
  152. package/packages/utility/src/tui/name-badge-state.ts +18 -2
  153. package/packages/utility/src/tui/name-badge.ts +5 -4
  154. package/packages/utility/src/tui/util-settings-tui.ts +41 -224
  155. package/packages/web-api/package.json +2 -6
  156. package/packages/web-api/src/engine/dependencies.ts +13 -134
  157. package/packages/web-api/src/engine/extract.ts +9 -99
  158. package/packages/web-api/src/engine/format.ts +0 -75
  159. package/packages/web-api/src/engine/types.ts +0 -39
  160. package/packages/web-api/src/index.ts +1 -2
  161. package/packages/web-api/src/providers/base.ts +0 -10
  162. package/packages/web-api/src/providers/duckduckgo.ts +0 -1
  163. package/packages/web-api/src/providers/firecrawl.ts +0 -9
  164. package/packages/web-api/src/providers/jina-reader.ts +0 -9
  165. package/packages/web-api/src/providers/jina-search.ts +0 -9
  166. package/packages/web-api/src/providers/perplexity.ts +0 -9
  167. package/packages/web-api/src/providers/registry.ts +0 -80
  168. package/packages/web-api/src/providers/serpapi.ts +0 -9
  169. package/packages/web-api/src/providers/tavily.ts +0 -9
  170. package/packages/web-api/src/providers/wigolo-client.ts +0 -30
  171. package/packages/web-api/src/providers/wigolo.ts +0 -1
  172. package/packages/web-api/src/settings.ts +17 -77
  173. package/packages/web-api/src/tools.ts +21 -24
  174. package/packages/web-api/src/tui/settings-dialog.ts +0 -1
  175. package/packages/workflow/commands.ts +38 -122
  176. package/packages/workflow/index.ts +1 -5
  177. package/packages/workflow/package.json +2 -2
  178. package/packages/cocoindex/README.md +0 -94
  179. package/packages/cocoindex/bridge.ts +0 -842
  180. package/packages/cocoindex/commands.ts +0 -175
  181. package/packages/cocoindex/index.ts +0 -48
  182. package/packages/cocoindex/installer.ts +0 -397
  183. package/packages/cocoindex/package.json +0 -49
  184. package/packages/cocoindex/skills/cocoindex/SKILL.md +0 -88
  185. package/packages/cocoindex/tools.ts +0 -157
  186. package/packages/compactor/src/display/diff-renderer.ts +0 -281
  187. package/packages/compactor/src/display/line-width-safety.ts +0 -28
  188. package/packages/compactor/src/display/render-utils.ts +0 -52
  189. package/packages/compactor/src/display/thinking-label.ts +0 -18
  190. package/packages/compactor/src/display/tool-overrides.ts +0 -136
  191. package/packages/compactor/src/tools/compact.ts +0 -20
  192. package/packages/subagents/src/file-lock.ts +0 -102
  193. package/packages/utility/src/cache/ttl-cache.ts +0 -311
  194. package/packages/utility/src/diff/highlighter.ts +0 -352
  195. package/packages/utility/src/diff/parser.ts +0 -191
  196. package/packages/utility/src/diff/renderer.ts +0 -395
  197. package/packages/utility/src/diff/theme.ts +0 -316
  198. package/packages/utility/src/diff/wrapper.ts +0 -339
  199. package/packages/utility/src/display/capabilities.ts +0 -214
  200. package/packages/utility/src/display/width.ts +0 -226
  201. package/packages/utility/src/tools/batch.ts +0 -229
  202. package/packages/utility/src/tui/badge-settings-tui.ts +0 -388
  203. package/packages/utility/src/tui/badge-settings.ts +0 -67
  204. package/packages/utility/src/tui/settings-inspector.ts +0 -303
  205. package/packages/web-api/src/providers/llm-summarize.ts +0 -71
@@ -1,842 +0,0 @@
1
- /**
2
- * bridge.ts — CocoIndex CLI interaction layer
3
- *
4
- * Spawns cocoindex commands and queries LanceDB directly for search.
5
- * The bridge handles:
6
- * - CLI detection (is cocoindex installed?)
7
- * - Pipeline initialization (scaffold main.py)
8
- * - Project indexing (cocoindex update)
9
- * - Status reporting (last run, doc count)
10
- * - Search (query LanceDB directly via Node.js SDK)
11
- */
12
-
13
- import { execFileSync, spawn } from "node:child_process";
14
- import { existsSync, mkdirSync, readFileSync, writeFileSync, readdirSync, statSync } from "node:fs";
15
- import { homedir } from "node:os";
16
- import { join, dirname } from "node:path";
17
- import { COCOINDEX_MIN_VERSION } from "@pi-unipi/core";
18
-
19
- // ─────────────────────────────────────────────────────────
20
- // Types
21
- // ─────────────────────────────────────────────────────────
22
-
23
- export interface IndexResult {
24
- success: boolean;
25
- chunksProcessed: number;
26
- durationMs: number;
27
- error?: string;
28
- }
29
-
30
- export interface StatusInfo {
31
- indexed: boolean;
32
- lastRun: string | null;
33
- docCount: number;
34
- pipelineConfigured: boolean;
35
- cliAvailable: boolean;
36
- targetStore: "lancedb" | "postgres" | "qdrant" | "sqlite" | "unknown";
37
- }
38
-
39
- export interface SearchOptions {
40
- limit?: number;
41
- offset?: number;
42
- minScore?: number;
43
- }
44
-
45
- export interface SearchResult {
46
- title: string;
47
- content: string;
48
- source: string;
49
- rank: number;
50
- contentType: "code" | "prose";
51
- matchLayer: "vector" | "fulltext" | "hybrid";
52
- }
53
-
54
- export interface CocoindexDeps {
55
- projectDir: string;
56
- pipelineDir: string;
57
- initialized: boolean;
58
- }
59
-
60
- // ─────────────────────────────────────────────────────────
61
- // Constants
62
- // ─────────────────────────────────────────────────────────
63
-
64
- const COCOINDEX_STATE_DIR = ".cocoindex";
65
- const DEFAULT_PIPELINE_DIR = ".unipi/cocoindex";
66
- const DEFAULT_LANCEDB_PATH = ".unipi/cocoindex/.lancedb";
67
- const DEFAULT_UPDATE_TIMEOUT_MS = 15 * 60 * 1000;
68
- const DEFAULT_LEXICAL_SCAN_LIMIT = 50_000;
69
-
70
- // ─────────────────────────────────────────────────────────
71
- // CLI Detection
72
- // ─────────────────────────────────────────────────────────
73
-
74
- let cachedAvailable: boolean | null = null;
75
-
76
- export interface AvailabilityOptions {
77
- /** Bypass the cached availability result. */
78
- useCache?: boolean;
79
- }
80
-
81
- /** Extract a semver-ish version from `cocoindex --version` output. */
82
- export function parseVersion(versionStr: string): string | null {
83
- const match = versionStr.match(/(?:^|[^0-9])(\d+\.\d+(?:\.\d+)?)(?:[^0-9]|$)/);
84
- return match?.[1] ?? null;
85
- }
86
-
87
- /** Compare semver-ish strings. Missing/invalid versions are not acceptable. */
88
- export function isVersionAtLeast(version: string | null | undefined, minimum = COCOINDEX_MIN_VERSION): boolean {
89
- const parsed = version ? parseVersion(version) : null;
90
- const parsedMinimum = parseVersion(minimum);
91
- if (!parsed || !parsedMinimum) return false;
92
-
93
- const actualParts = parsed.split(".").map((part) => Number.parseInt(part, 10));
94
- const minParts = parsedMinimum.split(".").map((part) => Number.parseInt(part, 10));
95
- const len = Math.max(actualParts.length, minParts.length, 3);
96
-
97
- for (let i = 0; i < len; i++) {
98
- const actual = actualParts[i] ?? 0;
99
- const min = minParts[i] ?? 0;
100
- if (!Number.isFinite(actual) || !Number.isFinite(min)) return false;
101
- if (actual > min) return true;
102
- if (actual < min) return false;
103
- }
104
- return true;
105
- }
106
-
107
- /** Reset cached availability, used after installer mutations. */
108
- export function resetAvailabilityCache(): void {
109
- cachedAvailable = null;
110
- }
111
-
112
- /** Check if cocoindex CLI is installed and available. */
113
- export async function isAvailable(options: AvailabilityOptions = {}): Promise<boolean> {
114
- const useCache = options.useCache ?? true;
115
- if (useCache && cachedAvailable !== null) return cachedAvailable;
116
-
117
- let available = false;
118
- try {
119
- const result = execFileSync(getCocoindexBinPath(), ["--version"], {
120
- encoding: "utf-8",
121
- timeout: 5000,
122
- stdio: ["pipe", "pipe", "pipe"],
123
- });
124
- available = result.trim().length > 0;
125
- } catch {
126
- available = false;
127
- }
128
-
129
- if (useCache) cachedAvailable = available;
130
- return available;
131
- }
132
-
133
- /** Get cocoindex CLI version string. */
134
- export async function getVersion(): Promise<string | null> {
135
- try {
136
- const result = execFileSync(getCocoindexBinPath(), ["--version"], {
137
- encoding: "utf-8",
138
- timeout: 5000,
139
- stdio: ["pipe", "pipe", "pipe"],
140
- });
141
- return result.trim();
142
- } catch {
143
- return null;
144
- }
145
- }
146
-
147
- /** Resolve cocoindex binary path — checks PATH, uv tool bin path, then common mise locations. */
148
- export function getCocoindexBinPath(): string {
149
- return resolveCocoindexBin();
150
- }
151
-
152
- function resolveCocoindexBin(): string {
153
- // Try PATH first.
154
- try {
155
- const resolved = execFileSync("sh", ["-c", "command -v cocoindex"], {
156
- encoding: "utf-8",
157
- timeout: 3000,
158
- stdio: ["pipe", "pipe", "pipe"],
159
- }).trim();
160
- if (resolved) return resolved;
161
- } catch {
162
- // Not on PATH
163
- }
164
-
165
- // uv tool install exposes binaries here by default.
166
- const uvToolBin = join(homedir(), ".local", "bin", "cocoindex");
167
- if (existsSync(uvToolBin)) return uvToolBin;
168
-
169
- // Try mise python installations.
170
- const miseRoot = join(homedir(), ".local", "share", "mise", "installs", "python");
171
- try {
172
- const versions = readdirSync(miseRoot).sort().reverse();
173
- for (const ver of versions) {
174
- const binPath = join(miseRoot, ver, "bin", "cocoindex");
175
- if (existsSync(binPath)) return binPath;
176
- }
177
- } catch {
178
- // mise not installed or no python versions
179
- }
180
-
181
- return "cocoindex"; // Fall back to PATH resolution
182
- }
183
-
184
- // ─────────────────────────────────────────────────────────
185
- // Pipeline Management
186
- // ─────────────────────────────────────────────────────────
187
-
188
- /** Get the pipeline directory for a project. */
189
- export function getPipelineDir(projectDir: string): string {
190
- return join(projectDir, DEFAULT_PIPELINE_DIR);
191
- }
192
-
193
- /** Check if a pipeline is already initialized. */
194
- export async function isPipelineInitialized(pipelineDir: string): Promise<boolean> {
195
- return existsSync(join(pipelineDir, "main.py"));
196
- }
197
-
198
- /** Detect the target store from main.py content. */
199
- export function detectTargetStore(pipelineDir: string): StatusInfo["targetStore"] {
200
- const mainPyPath = join(pipelineDir, "main.py");
201
- if (!existsSync(mainPyPath)) return "unknown";
202
-
203
- try {
204
- const content = readFileSync(mainPyPath, "utf-8");
205
- if (content.includes("LanceDB") || content.includes("lancedb")) return "lancedb";
206
- if (content.includes("Postgres") || content.includes("postgresql")) return "postgres";
207
- if (content.includes("Qdrant") || content.includes("qdrant")) return "qdrant";
208
- if (content.includes("SQLite") || content.includes("sqlite")) return "sqlite";
209
- return "unknown";
210
- } catch {
211
- return "unknown";
212
- }
213
- }
214
-
215
- /** Initialize a cocoindex pipeline with default LanceDB target. */
216
- export async function initPipeline(projectDir: string): Promise<{ success: boolean; error?: string }> {
217
- const pipelineDir = getPipelineDir(projectDir);
218
-
219
- // Create directory
220
- if (!existsSync(pipelineDir)) {
221
- mkdirSync(pipelineDir, { recursive: true });
222
- }
223
-
224
- // Don't overwrite existing pipeline
225
- if (existsSync(join(pipelineDir, "main.py"))) {
226
- return { success: true };
227
- }
228
-
229
- // Read embedding config from memory settings
230
- const embeddingConfig = loadEmbeddingConfig();
231
-
232
- const template = generatePipelineTemplate(projectDir, embeddingConfig);
233
- writeFileSync(join(pipelineDir, "main.py"), template, "utf-8");
234
-
235
- return { success: true };
236
- }
237
-
238
- // ─────────────────────────────────────────────────────────
239
- // Indexing
240
- // ─────────────────────────────────────────────────────────
241
-
242
- /** Run cocoindex update to index the project. */
243
- export async function indexProject(projectDir: string): Promise<IndexResult> {
244
- const available = await isAvailable();
245
- if (!available) {
246
- return {
247
- success: false,
248
- chunksProcessed: 0,
249
- durationMs: 0,
250
- error: "CocoIndex CLI not found. Run /unipi:cocoindex-init to install cocoindex[lancedb]>=1.0.",
251
- };
252
- }
253
-
254
- const pipelineDir = getPipelineDir(projectDir);
255
- if (!existsSync(join(pipelineDir, "main.py"))) {
256
- return {
257
- success: false,
258
- chunksProcessed: 0,
259
- durationMs: 0,
260
- error: "Pipeline not initialized. Run /unipi:cocoindex-init first.",
261
- };
262
- }
263
-
264
- const start = Date.now();
265
-
266
- const cocoindexBin = resolveCocoindexBin();
267
-
268
- return new Promise<IndexResult>((resolve) => {
269
- const proc = spawn(cocoindexBin, ["update", "main.py"], {
270
- cwd: pipelineDir,
271
- stdio: ["pipe", "pipe", "pipe"],
272
- });
273
-
274
- let stdout = "";
275
- let stderr = "";
276
- let timedOut = false;
277
- let settled = false;
278
- const timeoutMs = getUpdateTimeoutMs();
279
-
280
- const timer = setTimeout(() => {
281
- timedOut = true;
282
- proc.kill("SIGTERM");
283
- setTimeout(() => {
284
- if (!settled) proc.kill("SIGKILL");
285
- }, 5000).unref();
286
- }, timeoutMs);
287
- timer.unref();
288
-
289
- proc.stdout.on("data", (data: Buffer) => {
290
- stdout += data.toString();
291
- });
292
-
293
- proc.stderr.on("data", (data: Buffer) => {
294
- stderr += data.toString();
295
- });
296
-
297
- proc.on("close", (code: number | null, signal: NodeJS.Signals | null) => {
298
- settled = true;
299
- clearTimeout(timer);
300
- const durationMs = Date.now() - start;
301
- const chunksProcessed = parseChunksProcessed(stdout);
302
-
303
- if (code === 0) {
304
- resolve({ success: true, chunksProcessed, durationMs });
305
- } else {
306
- resolve({
307
- success: false,
308
- chunksProcessed,
309
- durationMs,
310
- error: formatIndexFailure({ code, signal, timedOut, timeoutMs, stdout, stderr }),
311
- });
312
- }
313
- });
314
-
315
- proc.on("error", (err: Error) => {
316
- settled = true;
317
- clearTimeout(timer);
318
- resolve({
319
- success: false,
320
- chunksProcessed: 0,
321
- durationMs: Date.now() - start,
322
- error: err.message,
323
- });
324
- });
325
- });
326
- }
327
-
328
- /** Parse the number of files processed from cocoindex v1.0+ output. */
329
- function getUpdateTimeoutMs(): number {
330
- const raw = process.env.COCOINDEX_UPDATE_TIMEOUT_MS;
331
- if (!raw) return DEFAULT_UPDATE_TIMEOUT_MS;
332
-
333
- const parsed = Number(raw);
334
- if (!Number.isFinite(parsed) || parsed <= 0) return DEFAULT_UPDATE_TIMEOUT_MS;
335
- return parsed;
336
- }
337
-
338
- function formatIndexFailure(args: {
339
- code: number | null;
340
- signal: NodeJS.Signals | null;
341
- timedOut: boolean;
342
- timeoutMs: number;
343
- stdout: string;
344
- stderr: string;
345
- }): string {
346
- const parts: string[] = [];
347
-
348
- if (args.timedOut) {
349
- parts.push(`Timed out after ${(args.timeoutMs / 1000).toFixed(0)}s`);
350
- } else if (args.signal) {
351
- parts.push(`Process terminated by ${args.signal}`);
352
- } else {
353
- parts.push(`Process exited with code ${args.code ?? "unknown"}`);
354
- }
355
-
356
- const stderr = tailText(args.stderr.trim(), 4000);
357
- const stdout = tailText(args.stdout.trim(), 2000);
358
- if (stderr) parts.push(`stderr:\n${stderr}`);
359
- if (stdout) parts.push(`stdout:\n${stdout}`);
360
-
361
- return parts.join("\n\n");
362
- }
363
-
364
- function tailText(text: string, maxChars: number): string {
365
- if (text.length <= maxChars) return text;
366
- return `…${text.slice(-maxChars)}`;
367
- }
368
-
369
- function parseChunksProcessed(output: string): number {
370
- // v1.0+ format: "✅ process_file: 604 total | 604 added"
371
- // Capture the last "added" or "reprocessed" count for process_file
372
- const lines = output.split("\n");
373
- let lastProcessLine: string | undefined;
374
- for (const line of lines) {
375
- if (line.includes("process_file:") && (line.includes("added") || line.includes("reprocessed"))) {
376
- lastProcessLine = line;
377
- }
378
- }
379
- if (lastProcessLine) {
380
- // Prefer completed work counts. Lines can contain multiple counters, e.g.
381
- // "process_file: 615 total | 8 added, 606 reprocessed".
382
- const matches = [...lastProcessLine.matchAll(/(\d+)\s+(?:added|reprocessed|skipped|deleted)/g)];
383
- const completed = matches.reduce((sum, match) => sum + parseInt(match[1], 10), 0);
384
- if (completed > 0) return completed;
385
-
386
- const total = lastProcessLine.match(/process_file:\s*(\d+)\s+total/);
387
- if (total) return parseInt(total[1], 10);
388
- }
389
-
390
- // Fallback: old format "Processed 42 chunks"
391
- const fallback = output.match(/processed\s+(\d+)\s+chunks?/i)
392
- ?? output.match(/(\d+)\s+chunks?\s+processed/i)
393
- ?? output.match(/indexed\s+(\d+)/i);
394
- return fallback ? parseInt(fallback[1], 10) : 0;
395
- }
396
-
397
- // ─────────────────────────────────────────────────────────
398
- // Status
399
- // ─────────────────────────────────────────────────────────
400
-
401
- /** Get indexing status for the project. */
402
- export async function status(projectDir: string): Promise<StatusInfo> {
403
- const pipelineDir = getPipelineDir(projectDir);
404
- const cliAvailable = await isAvailable();
405
- const pipelineConfigured = existsSync(join(pipelineDir, "main.py"));
406
- const targetStore = detectTargetStore(pipelineDir);
407
-
408
- let docCount = 0;
409
- let lastRun: string | null = null;
410
-
411
- // Check LanceDB data for doc count and freshness
412
- const lancedbPath = join(pipelineDir, ".lancedb");
413
- if (existsSync(lancedbPath)) {
414
- try {
415
- const stat = statSync(lancedbPath);
416
- lastRun = stat.mtime.toISOString();
417
- // Count .lance files as a rough doc estimate
418
- const files = readdirSync(lancedbPath, { recursive: true });
419
- docCount = (files as string[]).filter((f) => f.endsWith(".lance")).length;
420
- } catch {
421
- // Non-fatal
422
- }
423
- }
424
-
425
- return {
426
- indexed: docCount > 0,
427
- lastRun,
428
- docCount,
429
- pipelineConfigured,
430
- cliAvailable,
431
- targetStore,
432
- };
433
- }
434
-
435
- // ─────────────────────────────────────────────────────────
436
- // Search
437
- // ─────────────────────────────────────────────────────────
438
-
439
- /** Search indexed content by querying LanceDB directly. */
440
- export async function search(
441
- projectDir: string,
442
- query: string,
443
- options?: SearchOptions,
444
- ): Promise<SearchResult[]> {
445
- const limit = options?.limit ?? 10;
446
- const offset = options?.offset ?? 0;
447
-
448
- try {
449
- const lancedbPath = join(getPipelineDir(projectDir), ".lancedb");
450
- if (!existsSync(lancedbPath)) {
451
- return [];
452
- }
453
-
454
- // Dynamic import — LanceDB SDK may not be installed.
455
- // @ts-ignore — optional dependency, may not be installed
456
- const lancedb = await import("@lancedb/lancedb");
457
- const db = await lancedb.connect(lancedbPath);
458
-
459
- const tableNames = await db.tableNames();
460
- if (tableNames.length === 0) return [];
461
-
462
- const table = await db.openTable(tableNames[0]);
463
-
464
- // Prefer semantic vector search when the pipeline/table provides a vector
465
- // column. Older generated pipelines only contain path/chunk_index/content;
466
- // LanceDB throws for those tables, so continue to FTS/lexical fallback.
467
- const queryVector = await generateQueryEmbedding(query);
468
- if (queryVector) {
469
- const vectorResults = await vectorSearch(table, queryVector, limit, offset);
470
- if (vectorResults.length > 0) return vectorResults;
471
- }
472
-
473
- // Prefer LanceDB's native FTS when an inverted index exists on content.
474
- const ftsResults = await fullTextSearch(table, query, limit, offset);
475
- if (ftsResults.length > 0) return ftsResults;
476
-
477
- // Last-resort compatibility path for existing text-only LanceDB tables.
478
- // This keeps indexed projects searchable immediately instead of returning
479
- // a misleading "run cocoindex-update" message when no vector/FTS index is
480
- // available yet.
481
- return lexicalSearch(table, query, limit, offset);
482
- } catch (err: any) {
483
- if (err?.code === "MODULE_NOT_FOUND" || err?.message?.includes("Cannot find module")) {
484
- return [{
485
- title: "Search Unavailable",
486
- content: "LanceDB SDK not installed. Install with: npm install @lancedb/lancedb",
487
- source: "",
488
- rank: 0,
489
- contentType: "prose",
490
- matchLayer: "fulltext",
491
- }];
492
- }
493
- return [{
494
- title: "Search Error",
495
- content: `CocoIndex LanceDB search failed: ${err?.message ?? String(err)}`,
496
- source: "",
497
- rank: 0,
498
- contentType: "prose",
499
- matchLayer: "fulltext",
500
- }];
501
- }
502
- }
503
-
504
- async function vectorSearch(table: any, queryVector: number[], limit: number, offset: number): Promise<SearchResult[]> {
505
- try {
506
- const results = await table.search(queryVector)
507
- .limit(limit + offset)
508
- .toArray();
509
-
510
- return results.slice(offset).map((r: any, i: number) => rowToSearchResult(r, i, "vector"));
511
- } catch {
512
- return [];
513
- }
514
- }
515
-
516
- /** Fallback full-text search when vector search isn't available. */
517
- async function fullTextSearch(table: any, query: string, limit: number, offset: number): Promise<SearchResult[]> {
518
- try {
519
- const results = await table.search(query, "fts")
520
- .limit(limit + offset)
521
- .toArray();
522
-
523
- return results.slice(offset).map((r: any, i: number) => rowToSearchResult(r, i, "fulltext"));
524
- } catch {
525
- return [];
526
- }
527
- }
528
-
529
- /**
530
- * Compatibility fallback for existing LanceDB tables that contain text chunks
531
- * but no vector column or full-text inverted index.
532
- */
533
- async function lexicalSearch(table: any, query: string, limit: number, offset: number): Promise<SearchResult[]> {
534
- try {
535
- const terms = tokenize(query);
536
- if (terms.length === 0) return [];
537
-
538
- const rows = await table.query()
539
- .limit(DEFAULT_LEXICAL_SCAN_LIMIT)
540
- .toArray();
541
-
542
- const phrase = query.trim().toLowerCase();
543
- const scored = rows
544
- .map((row: any) => {
545
- const content = String(row.content ?? row.text ?? "");
546
- const path = String(row.path ?? row.source ?? "");
547
- const haystack = `${path}\n${content}`.toLowerCase();
548
- let score = 0;
549
-
550
- for (const term of terms) {
551
- const contentMatches = countOccurrences(content.toLowerCase(), term);
552
- const pathMatches = countOccurrences(path.toLowerCase(), term);
553
- score += contentMatches + pathMatches * 3;
554
- }
555
-
556
- if (phrase && haystack.includes(phrase)) score += terms.length * 4;
557
- return { row, score };
558
- })
559
- .filter((item: { score: number }) => item.score > 0)
560
- .sort((a: { score: number }, b: { score: number }) => b.score - a.score)
561
- .slice(offset, offset + limit);
562
-
563
- return scored.map((item: { row: any; score: number }, i: number) => ({
564
- ...rowToSearchResult(item.row, i, "fulltext"),
565
- rank: item.score,
566
- }));
567
- } catch {
568
- return [];
569
- }
570
- }
571
-
572
- function rowToSearchResult(r: any, i: number, matchLayer: SearchResult["matchLayer"]): SearchResult {
573
- const path = r.path ?? r.source ?? "";
574
- return {
575
- title: r.title ?? path ?? `Result ${i + 1}`,
576
- content: r.content ?? r.text ?? String(r),
577
- source: r.source ?? path ?? "",
578
- rank: r._distance ?? (1 - (r.score ?? 0)),
579
- contentType: (r.content_type === "code" || path?.match(/\.(ts|tsx|js|jsx|py|rs|go|sh|bash)$/)) ? "code" : "prose",
580
- matchLayer,
581
- };
582
- }
583
-
584
- function tokenize(query: string): string[] {
585
- const seen = new Set<string>();
586
- const stopwords = new Set(["a", "an", "and", "are", "as", "at", "for", "from", "how", "in", "is", "of", "on", "or", "the", "to", "with"]);
587
- const terms = query
588
- .toLowerCase()
589
- .split(/[^a-z0-9_+#.-]+/i)
590
- .map((term) => term.trim())
591
- .filter((term) => term.length > 1 && !stopwords.has(term));
592
-
593
- return terms.filter((term) => {
594
- if (seen.has(term)) return false;
595
- seen.add(term);
596
- return true;
597
- });
598
- }
599
-
600
- function countOccurrences(value: string, needle: string): number {
601
- if (!needle) return 0;
602
- let count = 0;
603
- let index = value.indexOf(needle);
604
- while (index !== -1) {
605
- count += 1;
606
- index = value.indexOf(needle, index + needle.length);
607
- }
608
- return count;
609
- }
610
-
611
- // ─────────────────────────────────────────────────────────
612
- // Embedding
613
- // ─────────────────────────────────────────────────────────
614
-
615
- interface EmbeddingConfig {
616
- apiKey: string | null;
617
- model: string;
618
- baseUrl: string;
619
- }
620
-
621
- /** Load embedding config — env var takes priority, then config file, then defaults. */
622
- function loadEmbeddingConfig(): EmbeddingConfig {
623
- // Env var takes top priority
624
- const envKey = process.env.OPENROUTER_API_KEY ?? null;
625
-
626
- const configPath = join(homedir(), ".unipi", "memory", "config.json");
627
- try {
628
- if (existsSync(configPath)) {
629
- const raw = readFileSync(configPath, "utf-8");
630
- const config = JSON.parse(raw);
631
- return {
632
- apiKey: envKey ?? config.openrouterApiKey ?? config.apiKey ?? null,
633
- model: config.embeddingModel ?? "qwen/qwen3-embedding-8b",
634
- baseUrl: config.openrouterBaseUrl ?? "https://openrouter.ai/api/v1",
635
- };
636
- }
637
- } catch {
638
- // Fall through to defaults
639
- }
640
- return {
641
- apiKey: envKey,
642
- model: "qwen/qwen3-embedding-8b",
643
- baseUrl: "https://openrouter.ai/api/v1",
644
- };
645
- }
646
-
647
- /** Generate embedding for a query using OpenRouter API. */
648
- async function generateQueryEmbedding(query: string): Promise<number[] | null> {
649
- const config = loadEmbeddingConfig();
650
- if (!config.apiKey) return null;
651
-
652
- try {
653
- const response = await fetch(`${config.baseUrl}/embeddings`, {
654
- method: "POST",
655
- headers: {
656
- "Authorization": `Bearer ${config.apiKey}`,
657
- "Content-Type": "application/json",
658
- },
659
- body: JSON.stringify({
660
- model: config.model,
661
- input: query,
662
- }),
663
- });
664
-
665
- if (!response.ok) return null;
666
-
667
- const data = await response.json() as any;
668
- return data.data?.[0]?.embedding ?? null;
669
- } catch {
670
- return null;
671
- }
672
- }
673
-
674
- // ─────────────────────────────────────────────────────────
675
- // Pipeline Template
676
- // ─────────────────────────────────────────────────────────
677
-
678
- /** Generate a cocoindex pipeline main.py template (v1.0+ API). */
679
- function generatePipelineTemplate(projectDir: string, embeddingConfig: EmbeddingConfig): string {
680
- const projectBasename = projectDir.split("/").pop() ?? "project";
681
- return `"""
682
- CocoIndex pipeline for ${projectBasename}
683
- Auto-generated by @pi-unipi/cocoindex — customize as needed.
684
- Requires cocoindex >= 1.0.
685
- """
686
- import pathlib
687
- from dataclasses import dataclass
688
- from typing import AsyncIterator
689
-
690
- import cocoindex as coco
691
- from cocoindex.connectors import localfs, lancedb
692
- from cocoindex.resources.file import PatternFilePathMatcher
693
-
694
- import os
695
-
696
- # ── Configuration ────────────────────────────────────
697
- PROJECT_ROOT = os.environ.get("PROJECT_ROOT", "${projectDir}")
698
- # Safety limit for huge generated/lock files. Set COCO_MAX_FILE_CHARS=0 to disable.
699
- MAX_FILE_CHARS = int(os.environ.get("COCO_MAX_FILE_CHARS", "200000"))
700
-
701
- # ── LanceDB context key ──────────────────────────────
702
- db_key = coco.ContextKey("lancedb/${projectBasename}")
703
-
704
-
705
- # ── Environment setup (async lifespan) ───────────────
706
- @coco.lifespan
707
- async def coco_lifespan(builder: coco.EnvironmentBuilder) -> AsyncIterator[None]:
708
- """Configure environment: DB path + LanceDB connection."""
709
- builder.settings.db_path = pathlib.Path(__file__).parent / "cocoindex.db"
710
-
711
- db_path = pathlib.Path(__file__).parent / ".lancedb"
712
- conn = await lancedb.connect_async(str(db_path))
713
- builder.provide(db_key, conn)
714
-
715
- yield
716
-
717
-
718
- # ── Row type for LanceDB ─────────────────────────────
719
- @dataclass
720
- class IndexRow:
721
- """A single indexed chunk stored in LanceDB."""
722
- path: str
723
- chunk_index: int
724
- content: str
725
-
726
-
727
- # ── Chunking function (memoized) ─────────────────────
728
- @coco.fn
729
- async def chunk_text(
730
- content: str,
731
- *,
732
- chunk_size: int = 1500,
733
- chunk_overlap: int = 200,
734
- ) -> list[tuple[int, str]]:
735
- """Split text into overlapping chunks."""
736
- if not content.strip():
737
- return []
738
-
739
- chunks: list[tuple[int, str]] = []
740
- start = 0
741
- idx = 0
742
- while start < len(content):
743
- end = min(start + chunk_size, len(content))
744
- chunk = content[start:end].strip()
745
- if chunk:
746
- chunks.append((idx, chunk))
747
- idx += 1
748
- start += chunk_size - chunk_overlap
749
- if start < 0:
750
- start = 0
751
-
752
- return chunks
753
-
754
-
755
- # ── Process a single file ────────────────────────────
756
- @coco.fn
757
- async def process_file(
758
- file: localfs.File,
759
- table: lancedb.TableTarget,
760
- ) -> None:
761
- """Read a file, chunk it, and declare rows in LanceDB."""
762
- try:
763
- content = await file.read_text()
764
- except Exception:
765
- return
766
-
767
- if not content.strip():
768
- return
769
- if MAX_FILE_CHARS > 0 and len(content) > MAX_FILE_CHARS:
770
- return
771
-
772
- relative = file.file_path.path.as_posix()
773
- chunks = await chunk_text(content)
774
-
775
- for chunk_idx, text in chunks:
776
- table.declare_row(row=IndexRow(
777
- path=relative,
778
- chunk_index=chunk_idx,
779
- content=text,
780
- ))
781
-
782
-
783
- # ── Main app function ────────────────────────────────
784
- @coco.fn
785
- async def app_main() -> None:
786
- """Walk project files -> chunk -> store in LanceDB."""
787
- project_root = pathlib.Path(PROJECT_ROOT)
788
-
789
- # 1) Declare LanceDB table target
790
- table_schema = await lancedb.TableSchema.from_class(
791
- IndexRow,
792
- primary_key=["path", "chunk_index"],
793
- )
794
-
795
- target = await coco.mount_target(
796
- lancedb.table_target(
797
- db_key,
798
- "${projectBasename}_index",
799
- table_schema,
800
- ),
801
- )
802
- table = lancedb.TableTarget(target, table_schema)
803
-
804
- # 2) Walk project files
805
- walker = localfs.walk_dir(
806
- project_root,
807
- recursive=True,
808
- path_matcher=PatternFilePathMatcher(
809
- included_patterns=[
810
- "**/*.ts", "**/*.tsx", "**/*.js", "**/*.jsx",
811
- "**/*.py", "**/*.rs", "**/*.go",
812
- "**/*.md", "**/*.txt", "**/*.json", "**/*.yaml", "**/*.yml",
813
- "**/*.sh", "**/*.bash",
814
- ],
815
- excluded_patterns=[
816
- "**/node_modules/**", "**/.git/**", "**/dist/**",
817
- "**/build/**", "**/.next/**", "**/__pycache__/**",
818
- "**/coverage/**", "**/.turbo/**", "**/.cache/**",
819
- "**/.unipi/**",
820
- "**/*.min.js", "**/bundled.js", "**/bundle.js", "**/*bundle*.js",
821
- "**/package-lock.json", "**/pnpm-lock.yaml", "**/yarn.lock",
822
- ],
823
- ),
824
- )
825
-
826
- # 3) Process each file
827
- async for file in walker:
828
- await coco.mount(
829
- coco.component_subpath("process", file.file_path.path.as_posix()),
830
- process_file,
831
- file,
832
- table,
833
- )
834
-
835
-
836
- # ── App instance (required by CLI) ───────────────────
837
- app = coco.App(
838
- coco.AppConfig(name="local_${projectBasename}"),
839
- app_main,
840
- )
841
- `;
842
- }