bluera-knowledge 0.9.32 → 0.9.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/hooks/post-edit-check.sh +5 -3
- package/.claude/skills/atomic-commits/SKILL.md +3 -1
- package/.husky/pre-commit +3 -2
- package/.prettierrc +9 -0
- package/.versionrc.json +1 -1
- package/CHANGELOG.md +70 -0
- package/CLAUDE.md +6 -0
- package/README.md +25 -13
- package/bun.lock +277 -33
- package/dist/{chunk-L2YVNC63.js → chunk-6FHWC36B.js} +9 -1
- package/dist/chunk-6FHWC36B.js.map +1 -0
- package/dist/{chunk-RST4XGRL.js → chunk-DC7CGSGT.js} +288 -241
- package/dist/chunk-DC7CGSGT.js.map +1 -0
- package/dist/{chunk-6PBP5DVD.js → chunk-WFNPNAAP.js} +3212 -3054
- package/dist/chunk-WFNPNAAP.js.map +1 -0
- package/dist/{chunk-WT2DAEO7.js → chunk-Z2KKVH45.js} +548 -482
- package/dist/chunk-Z2KKVH45.js.map +1 -0
- package/dist/index.js +871 -758
- package/dist/index.js.map +1 -1
- package/dist/mcp/server.js +3 -3
- package/dist/watch.service-BJV3TI3F.js +7 -0
- package/dist/workers/background-worker-cli.js +97 -71
- package/dist/workers/background-worker-cli.js.map +1 -1
- package/eslint.config.js +43 -1
- package/package.json +18 -11
- package/plugin.json +8 -0
- package/python/requirements.txt +1 -1
- package/src/analysis/ast-parser.test.ts +12 -11
- package/src/analysis/ast-parser.ts +28 -22
- package/src/analysis/code-graph.test.ts +52 -62
- package/src/analysis/code-graph.ts +9 -13
- package/src/analysis/dependency-usage-analyzer.test.ts +91 -271
- package/src/analysis/dependency-usage-analyzer.ts +52 -24
- package/src/analysis/go-ast-parser.test.ts +22 -22
- package/src/analysis/go-ast-parser.ts +18 -25
- package/src/analysis/parser-factory.test.ts +9 -9
- package/src/analysis/parser-factory.ts +3 -3
- package/src/analysis/python-ast-parser.test.ts +27 -27
- package/src/analysis/python-ast-parser.ts +2 -2
- package/src/analysis/repo-url-resolver.test.ts +82 -82
- package/src/analysis/rust-ast-parser.test.ts +19 -19
- package/src/analysis/rust-ast-parser.ts +17 -27
- package/src/analysis/tree-sitter-parser.test.ts +3 -3
- package/src/analysis/tree-sitter-parser.ts +10 -16
- package/src/cli/commands/crawl.test.ts +40 -24
- package/src/cli/commands/crawl.ts +186 -166
- package/src/cli/commands/index-cmd.test.ts +90 -90
- package/src/cli/commands/index-cmd.ts +52 -36
- package/src/cli/commands/mcp.test.ts +6 -6
- package/src/cli/commands/mcp.ts +2 -2
- package/src/cli/commands/plugin-api.test.ts +16 -18
- package/src/cli/commands/plugin-api.ts +9 -6
- package/src/cli/commands/search.test.ts +16 -7
- package/src/cli/commands/search.ts +124 -87
- package/src/cli/commands/serve.test.ts +67 -25
- package/src/cli/commands/serve.ts +18 -3
- package/src/cli/commands/setup.test.ts +176 -101
- package/src/cli/commands/setup.ts +140 -117
- package/src/cli/commands/store.test.ts +82 -53
- package/src/cli/commands/store.ts +56 -37
- package/src/cli/program.ts +2 -2
- package/src/crawl/article-converter.test.ts +4 -1
- package/src/crawl/article-converter.ts +46 -31
- package/src/crawl/bridge.test.ts +240 -132
- package/src/crawl/bridge.ts +87 -30
- package/src/crawl/claude-client.test.ts +124 -56
- package/src/crawl/claude-client.ts +7 -15
- package/src/crawl/intelligent-crawler.test.ts +65 -22
- package/src/crawl/intelligent-crawler.ts +86 -53
- package/src/crawl/markdown-utils.ts +1 -4
- package/src/db/embeddings.ts +4 -6
- package/src/db/lance.test.ts +4 -4
- package/src/db/lance.ts +16 -12
- package/src/index.ts +26 -17
- package/src/logging/index.ts +1 -5
- package/src/logging/logger.ts +3 -5
- package/src/logging/payload.test.ts +1 -1
- package/src/logging/payload.ts +3 -5
- package/src/mcp/commands/index.ts +2 -2
- package/src/mcp/commands/job.commands.ts +12 -18
- package/src/mcp/commands/meta.commands.ts +13 -13
- package/src/mcp/commands/registry.ts +5 -8
- package/src/mcp/commands/store.commands.ts +19 -19
- package/src/mcp/handlers/execute.handler.test.ts +10 -10
- package/src/mcp/handlers/execute.handler.ts +4 -5
- package/src/mcp/handlers/index.ts +10 -14
- package/src/mcp/handlers/job.handler.test.ts +10 -10
- package/src/mcp/handlers/job.handler.ts +22 -25
- package/src/mcp/handlers/search.handler.test.ts +36 -65
- package/src/mcp/handlers/search.handler.ts +135 -104
- package/src/mcp/handlers/store.handler.test.ts +41 -52
- package/src/mcp/handlers/store.handler.ts +108 -88
- package/src/mcp/schemas/index.test.ts +73 -68
- package/src/mcp/schemas/index.ts +18 -12
- package/src/mcp/server.test.ts +1 -1
- package/src/mcp/server.ts +59 -46
- package/src/plugin/commands.test.ts +230 -95
- package/src/plugin/commands.ts +24 -25
- package/src/plugin/dependency-analyzer.test.ts +52 -52
- package/src/plugin/dependency-analyzer.ts +85 -22
- package/src/plugin/git-clone.test.ts +24 -13
- package/src/plugin/git-clone.ts +3 -7
- package/src/server/app.test.ts +109 -109
- package/src/server/app.ts +32 -23
- package/src/server/index.test.ts +64 -66
- package/src/services/chunking.service.test.ts +32 -32
- package/src/services/chunking.service.ts +16 -9
- package/src/services/code-graph.service.test.ts +30 -36
- package/src/services/code-graph.service.ts +24 -10
- package/src/services/code-unit.service.test.ts +55 -11
- package/src/services/code-unit.service.ts +85 -11
- package/src/services/config.service.test.ts +37 -18
- package/src/services/config.service.ts +30 -7
- package/src/services/index.service.test.ts +49 -18
- package/src/services/index.service.ts +98 -48
- package/src/services/index.ts +6 -9
- package/src/services/job.service.test.ts +22 -22
- package/src/services/job.service.ts +18 -18
- package/src/services/project-root.service.test.ts +1 -3
- package/src/services/search.service.test.ts +248 -120
- package/src/services/search.service.ts +286 -156
- package/src/services/services.test.ts +1 -1
- package/src/services/snippet.service.test.ts +14 -6
- package/src/services/snippet.service.ts +7 -5
- package/src/services/store.service.test.ts +68 -29
- package/src/services/store.service.ts +41 -12
- package/src/services/watch.service.test.ts +34 -14
- package/src/services/watch.service.ts +11 -1
- package/src/types/brands.test.ts +3 -1
- package/src/types/index.ts +2 -13
- package/src/types/search.ts +10 -8
- package/src/utils/type-guards.test.ts +20 -15
- package/src/utils/type-guards.ts +1 -1
- package/src/workers/background-worker-cli.ts +28 -30
- package/src/workers/background-worker.test.ts +54 -40
- package/src/workers/background-worker.ts +76 -60
- package/src/workers/pid-file.test.ts +167 -0
- package/src/workers/pid-file.ts +82 -0
- package/src/workers/spawn-worker.test.ts +22 -10
- package/src/workers/spawn-worker.ts +6 -6
- package/tests/analysis/ast-parser.test.ts +3 -3
- package/tests/analysis/code-graph.test.ts +5 -5
- package/tests/fixtures/code-snippets/api/error-handling.ts +4 -15
- package/tests/fixtures/code-snippets/api/rest-controller.ts +3 -9
- package/tests/fixtures/code-snippets/auth/jwt-auth.ts +5 -21
- package/tests/fixtures/code-snippets/auth/oauth-flow.ts +4 -4
- package/tests/fixtures/code-snippets/database/repository-pattern.ts +11 -3
- package/tests/fixtures/corpus/oss-repos/hono/src/adapter/aws-lambda/handler.ts +2 -2
- package/tests/fixtures/corpus/oss-repos/hono/src/adapter/cloudflare-pages/handler.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/adapter/cloudflare-workers/serve-static.ts +2 -2
- package/tests/fixtures/corpus/oss-repos/hono/src/client/client.ts +2 -2
- package/tests/fixtures/corpus/oss-repos/hono/src/client/types.ts +22 -20
- package/tests/fixtures/corpus/oss-repos/hono/src/context.ts +13 -10
- package/tests/fixtures/corpus/oss-repos/hono/src/helper/accepts/accepts.ts +10 -7
- package/tests/fixtures/corpus/oss-repos/hono/src/helper/adapter/index.ts +2 -2
- package/tests/fixtures/corpus/oss-repos/hono/src/helper/css/index.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/helper/factory/index.ts +16 -16
- package/tests/fixtures/corpus/oss-repos/hono/src/helper/ssg/ssg.ts +2 -2
- package/tests/fixtures/corpus/oss-repos/hono/src/hono-base.ts +3 -3
- package/tests/fixtures/corpus/oss-repos/hono/src/hono.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/jsx/dom/css.ts +2 -2
- package/tests/fixtures/corpus/oss-repos/hono/src/jsx/dom/intrinsic-element/components.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/jsx/dom/render.ts +7 -7
- package/tests/fixtures/corpus/oss-repos/hono/src/jsx/hooks/index.ts +3 -3
- package/tests/fixtures/corpus/oss-repos/hono/src/jsx/intrinsic-element/components.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/jsx/utils.ts +6 -6
- package/tests/fixtures/corpus/oss-repos/hono/src/middleware/jsx-renderer/index.ts +3 -3
- package/tests/fixtures/corpus/oss-repos/hono/src/middleware/serve-static/index.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/preset/quick.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/preset/tiny.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/router/pattern-router/router.ts +2 -2
- package/tests/fixtures/corpus/oss-repos/hono/src/router/reg-exp-router/node.ts +4 -4
- package/tests/fixtures/corpus/oss-repos/hono/src/router/reg-exp-router/router.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/router/trie-router/node.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/types.ts +166 -169
- package/tests/fixtures/corpus/oss-repos/hono/src/utils/body.ts +8 -8
- package/tests/fixtures/corpus/oss-repos/hono/src/utils/color.ts +3 -3
- package/tests/fixtures/corpus/oss-repos/hono/src/utils/cookie.ts +2 -2
- package/tests/fixtures/corpus/oss-repos/hono/src/utils/encode.ts +2 -2
- package/tests/fixtures/corpus/oss-repos/hono/src/utils/types.ts +30 -33
- package/tests/fixtures/corpus/oss-repos/hono/src/validator/validator.ts +2 -2
- package/tests/fixtures/test-server.ts +3 -2
- package/tests/helpers/performance-metrics.ts +8 -25
- package/tests/helpers/search-relevance.ts +14 -69
- package/tests/integration/cli-consistency.test.ts +6 -5
- package/tests/integration/python-bridge.test.ts +13 -3
- package/tests/mcp/server.test.ts +1 -1
- package/tests/services/code-unit.service.test.ts +48 -0
- package/tests/services/job.service.test.ts +124 -0
- package/tests/services/search.progressive-context.test.ts +2 -2
- package/.claude-plugin/plugin.json +0 -13
- package/dist/chunk-6PBP5DVD.js.map +0 -1
- package/dist/chunk-L2YVNC63.js.map +0 -1
- package/dist/chunk-RST4XGRL.js.map +0 -1
- package/dist/chunk-WT2DAEO7.js.map +0 -1
- package/dist/watch.service-YAIKKDCF.js +0 -7
- package/skills/atomic-commits/SKILL.md +0 -77
- /package/dist/{watch.service-YAIKKDCF.js.map → watch.service-BJV3TI3F.js.map} +0 -0
package/dist/index.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
runMCPServer
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-Z2KKVH45.js";
|
|
5
5
|
import {
|
|
6
6
|
IntelligentCrawler
|
|
7
|
-
} from "./chunk-
|
|
7
|
+
} from "./chunk-DC7CGSGT.js";
|
|
8
8
|
import {
|
|
9
9
|
ASTParser,
|
|
10
10
|
ChunkingService,
|
|
@@ -16,312 +16,238 @@ import {
|
|
|
16
16
|
err,
|
|
17
17
|
extractRepoName,
|
|
18
18
|
ok
|
|
19
|
-
} from "./chunk-
|
|
20
|
-
import "./chunk-
|
|
19
|
+
} from "./chunk-WFNPNAAP.js";
|
|
20
|
+
import "./chunk-6FHWC36B.js";
|
|
21
21
|
|
|
22
22
|
// src/index.ts
|
|
23
23
|
import { homedir as homedir2 } from "os";
|
|
24
24
|
import { join as join4 } from "path";
|
|
25
25
|
|
|
26
|
-
// src/cli/
|
|
26
|
+
// src/cli/commands/crawl.ts
|
|
27
|
+
import { createHash } from "crypto";
|
|
27
28
|
import { Command } from "commander";
|
|
28
|
-
import
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
quiet: opts.quiet,
|
|
53
|
-
verbose: opts.verbose
|
|
54
|
-
};
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
// src/cli/commands/store.ts
|
|
58
|
-
import { Command as Command2 } from "commander";
|
|
59
|
-
function createStoreCommand(getOptions) {
|
|
60
|
-
const store = new Command2("store").description("Manage knowledge stores (collections of indexed documents)");
|
|
61
|
-
store.command("list").description("Show all stores with their type (file/repo/web) and ID").option("-t, --type <type>", "Filter by type: file, repo, or web").action(async (options) => {
|
|
62
|
-
const globalOpts = getOptions();
|
|
63
|
-
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
64
|
-
try {
|
|
65
|
-
const stores = await services.store.list(options.type);
|
|
66
|
-
if (globalOpts.format === "json") {
|
|
67
|
-
console.log(JSON.stringify(stores, null, 2));
|
|
68
|
-
} else if (globalOpts.quiet === true) {
|
|
69
|
-
for (const s of stores) {
|
|
70
|
-
console.log(s.name);
|
|
29
|
+
import ora from "ora";
|
|
30
|
+
function createCrawlCommand(getOptions) {
|
|
31
|
+
return new Command("crawl").description("Crawl web pages with natural language control and index into store").argument("<url>", "URL to crawl").argument("<store>", "Target web store to add crawled content to").option(
|
|
32
|
+
"--crawl <instruction>",
|
|
33
|
+
'Natural language instruction for what to crawl (e.g., "all Getting Started pages")'
|
|
34
|
+
).option(
|
|
35
|
+
"--extract <instruction>",
|
|
36
|
+
'Natural language instruction for what to extract (e.g., "extract API references")'
|
|
37
|
+
).option("--simple", "Use simple BFS mode instead of intelligent crawling").option("--max-pages <number>", "Maximum number of pages to crawl", "50").option("--headless", "Use headless browser for JavaScript-rendered sites").action(
|
|
38
|
+
async (url, storeIdOrName, cmdOptions) => {
|
|
39
|
+
const globalOpts = getOptions();
|
|
40
|
+
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
41
|
+
let store;
|
|
42
|
+
let storeCreated = false;
|
|
43
|
+
const existingStore = await services.store.getByIdOrName(storeIdOrName);
|
|
44
|
+
if (!existingStore) {
|
|
45
|
+
const result = await services.store.create({
|
|
46
|
+
name: storeIdOrName,
|
|
47
|
+
type: "web",
|
|
48
|
+
url
|
|
49
|
+
});
|
|
50
|
+
if (!result.success) {
|
|
51
|
+
await destroyServices(services);
|
|
52
|
+
throw new Error(`Failed to create store: ${result.error.message}`);
|
|
71
53
|
}
|
|
72
|
-
|
|
73
|
-
if (
|
|
74
|
-
|
|
75
|
-
} else {
|
|
76
|
-
console.log("\nStores:\n");
|
|
77
|
-
for (const s of stores) {
|
|
78
|
-
console.log(` ${s.name} (${s.type}) - ${s.id}`);
|
|
79
|
-
}
|
|
80
|
-
console.log("");
|
|
54
|
+
const createdStore = result.data;
|
|
55
|
+
if (createdStore.type !== "web") {
|
|
56
|
+
throw new Error("Unexpected store type after creation");
|
|
81
57
|
}
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
});
|
|
87
|
-
store.command("create <name>").description("Create a new store pointing to a local path or URL").requiredOption("-t, --type <type>", "Store type: file (local dir), repo (git), web (crawled site)").requiredOption("-s, --source <path>", "Local path for file/repo stores, URL for web stores").option("-d, --description <desc>", "Optional description for the store").option("--tags <tags>", "Comma-separated tags for filtering").action(async (name, options) => {
|
|
88
|
-
const globalOpts = getOptions();
|
|
89
|
-
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
90
|
-
let exitCode = 0;
|
|
91
|
-
try {
|
|
92
|
-
const isUrl = options.source.startsWith("http://") || options.source.startsWith("https://");
|
|
93
|
-
const result = await services.store.create({
|
|
94
|
-
name,
|
|
95
|
-
type: options.type,
|
|
96
|
-
path: options.type === "file" || options.type === "repo" && !isUrl ? options.source : void 0,
|
|
97
|
-
url: options.type === "web" || options.type === "repo" && isUrl ? options.source : void 0,
|
|
98
|
-
description: options.description,
|
|
99
|
-
tags: options.tags?.split(",").map((t) => t.trim())
|
|
100
|
-
});
|
|
101
|
-
if (result.success) {
|
|
102
|
-
if (globalOpts.format === "json") {
|
|
103
|
-
console.log(JSON.stringify(result.data, null, 2));
|
|
104
|
-
} else {
|
|
105
|
-
console.log(`
|
|
106
|
-
Created store: ${result.data.name} (${result.data.id})
|
|
107
|
-
`);
|
|
58
|
+
store = createdStore;
|
|
59
|
+
storeCreated = true;
|
|
60
|
+
if (globalOpts.quiet !== true && globalOpts.format !== "json") {
|
|
61
|
+
console.log(`Created web store: ${store.name}`);
|
|
108
62
|
}
|
|
63
|
+
} else if (existingStore.type !== "web") {
|
|
64
|
+
await destroyServices(services);
|
|
65
|
+
throw new Error(
|
|
66
|
+
`Store "${storeIdOrName}" exists but is not a web store (type: ${existingStore.type})`
|
|
67
|
+
);
|
|
109
68
|
} else {
|
|
110
|
-
|
|
111
|
-
exitCode = 1;
|
|
112
|
-
}
|
|
113
|
-
} finally {
|
|
114
|
-
await destroyServices(services);
|
|
115
|
-
}
|
|
116
|
-
if (exitCode !== 0) {
|
|
117
|
-
process.exit(exitCode);
|
|
118
|
-
}
|
|
119
|
-
});
|
|
120
|
-
store.command("info <store>").description("Show store details: ID, type, path/URL, timestamps").action(async (storeIdOrName) => {
|
|
121
|
-
const globalOpts = getOptions();
|
|
122
|
-
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
123
|
-
try {
|
|
124
|
-
const s = await services.store.getByIdOrName(storeIdOrName);
|
|
125
|
-
if (s === void 0) {
|
|
126
|
-
console.error(`Error: Store not found: ${storeIdOrName}`);
|
|
127
|
-
process.exit(3);
|
|
128
|
-
}
|
|
129
|
-
if (globalOpts.format === "json") {
|
|
130
|
-
console.log(JSON.stringify(s, null, 2));
|
|
131
|
-
} else {
|
|
132
|
-
console.log(`
|
|
133
|
-
Store: ${s.name}`);
|
|
134
|
-
console.log(` ID: ${s.id}`);
|
|
135
|
-
console.log(` Type: ${s.type}`);
|
|
136
|
-
if ("path" in s) console.log(` Path: ${s.path}`);
|
|
137
|
-
if ("url" in s && s.url !== void 0) console.log(` URL: ${s.url}`);
|
|
138
|
-
if (s.description !== void 0) console.log(` Description: ${s.description}`);
|
|
139
|
-
console.log(` Created: ${s.createdAt.toISOString()}`);
|
|
140
|
-
console.log(` Updated: ${s.updatedAt.toISOString()}`);
|
|
141
|
-
console.log("");
|
|
69
|
+
store = existingStore;
|
|
142
70
|
}
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
const s = await services.store.getByIdOrName(storeIdOrName);
|
|
152
|
-
if (s === void 0) {
|
|
153
|
-
console.error(`Error: Store not found: ${storeIdOrName}`);
|
|
154
|
-
process.exit(3);
|
|
71
|
+
const maxPages = cmdOptions.maxPages !== void 0 ? parseInt(cmdOptions.maxPages) : 50;
|
|
72
|
+
const isInteractive = process.stdout.isTTY && globalOpts.quiet !== true && globalOpts.format !== "json";
|
|
73
|
+
let spinner;
|
|
74
|
+
if (isInteractive) {
|
|
75
|
+
const mode = cmdOptions.simple === true ? "simple" : "intelligent";
|
|
76
|
+
spinner = ora(`Crawling ${url} (${mode} mode)`).start();
|
|
77
|
+
} else if (globalOpts.quiet !== true && globalOpts.format !== "json") {
|
|
78
|
+
console.log(`Crawling ${url}`);
|
|
155
79
|
}
|
|
156
|
-
const
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
80
|
+
const crawler = new IntelligentCrawler();
|
|
81
|
+
const webChunker = ChunkingService.forContentType("web");
|
|
82
|
+
let pagesIndexed = 0;
|
|
83
|
+
let chunksCreated = 0;
|
|
84
|
+
let exitCode = 0;
|
|
85
|
+
crawler.on("progress", (progress) => {
|
|
86
|
+
if (spinner) {
|
|
87
|
+
if (progress.type === "strategy") {
|
|
88
|
+
spinner.text = progress.message ?? "Analyzing crawl strategy...";
|
|
89
|
+
} else if (progress.type === "page") {
|
|
90
|
+
const url2 = progress.currentUrl ?? "unknown";
|
|
91
|
+
spinner.text = `Crawling ${String(progress.pagesVisited + 1)}/${String(maxPages)} - ${url2}`;
|
|
92
|
+
} else if (progress.type === "extraction") {
|
|
93
|
+
const url2 = progress.currentUrl ?? "unknown";
|
|
94
|
+
spinner.text = `Extracting from ${url2}...`;
|
|
95
|
+
} else if (progress.type === "error" && progress.message !== void 0) {
|
|
96
|
+
spinner.warn(progress.message);
|
|
97
|
+
}
|
|
161
98
|
}
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
99
|
+
});
|
|
100
|
+
try {
|
|
101
|
+
await services.lance.initialize(store.id);
|
|
102
|
+
const docs = [];
|
|
103
|
+
for await (const result of crawler.crawl(url, {
|
|
104
|
+
...cmdOptions.crawl !== void 0 && { crawlInstruction: cmdOptions.crawl },
|
|
105
|
+
...cmdOptions.extract !== void 0 && { extractInstruction: cmdOptions.extract },
|
|
106
|
+
maxPages,
|
|
107
|
+
...cmdOptions.simple !== void 0 && { simple: cmdOptions.simple },
|
|
108
|
+
useHeadless: cmdOptions.headless ?? false
|
|
109
|
+
})) {
|
|
110
|
+
const contentToProcess = result.extracted ?? result.markdown;
|
|
111
|
+
const chunks = webChunker.chunk(contentToProcess, `${result.url}.md`);
|
|
112
|
+
const fileType = classifyWebContentType(result.url, result.title);
|
|
113
|
+
const urlHash = createHash("md5").update(result.url).digest("hex");
|
|
114
|
+
for (const chunk of chunks) {
|
|
115
|
+
const chunkId = chunks.length > 1 ? `${store.id}-${urlHash}-${String(chunk.chunkIndex)}` : `${store.id}-${urlHash}`;
|
|
116
|
+
const vector = await services.embeddings.embed(chunk.content);
|
|
117
|
+
docs.push({
|
|
118
|
+
id: createDocumentId(chunkId),
|
|
119
|
+
content: chunk.content,
|
|
120
|
+
vector,
|
|
121
|
+
metadata: {
|
|
122
|
+
type: chunks.length > 1 ? "chunk" : "web",
|
|
123
|
+
storeId: store.id,
|
|
124
|
+
url: result.url,
|
|
125
|
+
title: result.title,
|
|
126
|
+
extracted: result.extracted !== void 0,
|
|
127
|
+
depth: result.depth,
|
|
128
|
+
indexedAt: /* @__PURE__ */ new Date(),
|
|
129
|
+
fileType,
|
|
130
|
+
chunkIndex: chunk.chunkIndex,
|
|
131
|
+
totalChunks: chunk.totalChunks,
|
|
132
|
+
sectionHeader: chunk.sectionHeader
|
|
133
|
+
}
|
|
134
|
+
});
|
|
135
|
+
chunksCreated++;
|
|
136
|
+
}
|
|
137
|
+
pagesIndexed++;
|
|
174
138
|
}
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
console.log(`Deleted store: ${s.name}`);
|
|
179
|
-
} else {
|
|
180
|
-
console.error(`Error: ${result.error.message}`);
|
|
181
|
-
process.exit(1);
|
|
182
|
-
}
|
|
183
|
-
} finally {
|
|
184
|
-
await destroyServices(services);
|
|
185
|
-
}
|
|
186
|
-
});
|
|
187
|
-
return store;
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
// src/cli/commands/search.ts
|
|
191
|
-
import { Command as Command3 } from "commander";
|
|
192
|
-
function createSearchCommand(getOptions) {
|
|
193
|
-
const search = new Command3("search").description("Search indexed documents using vector similarity + full-text matching").argument("<query>", "Search query").option("-s, --stores <stores>", "Limit search to specific stores (comma-separated IDs or names)").option("-m, --mode <mode>", "vector (embeddings only), fts (text only), hybrid (both, default)", "hybrid").option("-n, --limit <count>", "Maximum results to return (default: 10)", "10").option("-t, --threshold <score>", "Minimum score 0-1; omit low-relevance results").option("--include-content", "Show full document content, not just preview snippet").option("--detail <level>", "Context detail: minimal, contextual, full (default: minimal)", "minimal").action(async (query, options) => {
|
|
194
|
-
const globalOpts = getOptions();
|
|
195
|
-
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
196
|
-
try {
|
|
197
|
-
let storeIds = (await services.store.list()).map((s) => s.id);
|
|
198
|
-
if (options.stores !== void 0) {
|
|
199
|
-
const requestedStores = options.stores.split(",").map((s) => s.trim());
|
|
200
|
-
const resolvedStores = [];
|
|
201
|
-
for (const requested of requestedStores) {
|
|
202
|
-
const store = await services.store.getByIdOrName(requested);
|
|
203
|
-
if (store !== void 0) {
|
|
204
|
-
resolvedStores.push(store.id);
|
|
205
|
-
} else {
|
|
206
|
-
console.error(`Error: Store not found: ${requested}`);
|
|
207
|
-
process.exit(3);
|
|
139
|
+
if (docs.length > 0) {
|
|
140
|
+
if (spinner) {
|
|
141
|
+
spinner.text = "Indexing documents...";
|
|
208
142
|
}
|
|
143
|
+
await services.lance.addDocuments(store.id, docs);
|
|
209
144
|
}
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
for (const r of results.results) {
|
|
232
|
-
const path = r.metadata.path ?? r.metadata.url ?? "unknown";
|
|
233
|
-
console.log(path);
|
|
145
|
+
const crawlResult = {
|
|
146
|
+
success: true,
|
|
147
|
+
store: store.name,
|
|
148
|
+
storeCreated,
|
|
149
|
+
url,
|
|
150
|
+
pagesCrawled: pagesIndexed,
|
|
151
|
+
chunksCreated,
|
|
152
|
+
mode: cmdOptions.simple === true ? "simple" : "intelligent",
|
|
153
|
+
hadCrawlInstruction: cmdOptions.crawl !== void 0,
|
|
154
|
+
hadExtractInstruction: cmdOptions.extract !== void 0
|
|
155
|
+
};
|
|
156
|
+
if (globalOpts.format === "json") {
|
|
157
|
+
console.log(JSON.stringify(crawlResult, null, 2));
|
|
158
|
+
} else if (spinner !== void 0) {
|
|
159
|
+
spinner.succeed(
|
|
160
|
+
`Crawled ${String(pagesIndexed)} pages, indexed ${String(chunksCreated)} chunks`
|
|
161
|
+
);
|
|
162
|
+
} else if (globalOpts.quiet !== true) {
|
|
163
|
+
console.log(
|
|
164
|
+
`Crawled ${String(pagesIndexed)} pages, indexed ${String(chunksCreated)} chunks`
|
|
165
|
+
);
|
|
234
166
|
}
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
`);
|
|
240
|
-
if (results.results.length === 0) {
|
|
241
|
-
console.log("No results found.\n");
|
|
167
|
+
} catch (error) {
|
|
168
|
+
const message = `Crawl failed: ${error instanceof Error ? error.message : String(error)}`;
|
|
169
|
+
if (spinner) {
|
|
170
|
+
spinner.fail(message);
|
|
242
171
|
} else {
|
|
243
|
-
|
|
244
|
-
const r = results.results[i];
|
|
245
|
-
if (r === void 0) continue;
|
|
246
|
-
if (r.summary) {
|
|
247
|
-
console.log(`${String(i + 1)}. [${r.score.toFixed(2)}] ${r.summary.type}: ${r.summary.name}`);
|
|
248
|
-
console.log(` ${r.summary.location}`);
|
|
249
|
-
console.log(` ${r.summary.purpose}`);
|
|
250
|
-
if (r.context && options.detail !== "minimal") {
|
|
251
|
-
console.log(` Imports: ${r.context.keyImports.slice(0, 3).join(", ")}`);
|
|
252
|
-
console.log(` Related: ${r.context.relatedConcepts.slice(0, 3).join(", ")}`);
|
|
253
|
-
}
|
|
254
|
-
console.log();
|
|
255
|
-
} else {
|
|
256
|
-
const path = r.metadata.path ?? r.metadata.url ?? "unknown";
|
|
257
|
-
console.log(`${String(i + 1)}. [${r.score.toFixed(2)}] ${path}`);
|
|
258
|
-
const preview = r.highlight ?? r.content.slice(0, 150).replace(/\n/g, " ") + (r.content.length > 150 ? "..." : "");
|
|
259
|
-
console.log(` ${preview}
|
|
260
|
-
`);
|
|
261
|
-
}
|
|
262
|
-
}
|
|
172
|
+
console.error(`Error: ${message}`);
|
|
263
173
|
}
|
|
174
|
+
exitCode = 6;
|
|
175
|
+
} finally {
|
|
176
|
+
await crawler.stop();
|
|
177
|
+
await destroyServices(services);
|
|
178
|
+
}
|
|
179
|
+
if (exitCode !== 0) {
|
|
180
|
+
process.exit(exitCode);
|
|
264
181
|
}
|
|
265
|
-
} finally {
|
|
266
|
-
await destroyServices(services);
|
|
267
182
|
}
|
|
268
|
-
|
|
269
|
-
return search;
|
|
183
|
+
);
|
|
270
184
|
}
|
|
271
185
|
|
|
272
186
|
// src/cli/commands/index-cmd.ts
|
|
273
|
-
import { Command as
|
|
274
|
-
import
|
|
187
|
+
import { Command as Command2 } from "commander";
|
|
188
|
+
import ora2 from "ora";
|
|
275
189
|
function createIndexCommand(getOptions) {
|
|
276
|
-
const index = new
|
|
190
|
+
const index = new Command2("index").description("Scan store files, chunk text, generate embeddings, save to LanceDB").argument("<store>", "Store ID or name").option("--force", "Re-index all files even if unchanged").action(async (storeIdOrName, _options) => {
|
|
277
191
|
const globalOpts = getOptions();
|
|
278
192
|
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
193
|
+
let exitCode = 0;
|
|
279
194
|
try {
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
let spinner;
|
|
287
|
-
if (isInteractive) {
|
|
288
|
-
spinner = ora(`Indexing store: ${store.name}`).start();
|
|
289
|
-
} else if (globalOpts.quiet !== true && globalOpts.format !== "json") {
|
|
290
|
-
console.log(`Indexing store: ${store.name}`);
|
|
291
|
-
}
|
|
292
|
-
await services.lance.initialize(store.id);
|
|
293
|
-
const result = await services.index.indexStore(store, (event) => {
|
|
294
|
-
if (event.type === "progress") {
|
|
295
|
-
if (spinner) {
|
|
296
|
-
spinner.text = `Indexing: ${String(event.current)}/${String(event.total)} files - ${event.message}`;
|
|
297
|
-
}
|
|
195
|
+
indexLogic: {
|
|
196
|
+
const store = await services.store.getByIdOrName(storeIdOrName);
|
|
197
|
+
if (store === void 0) {
|
|
198
|
+
console.error(`Error: Store not found: ${storeIdOrName}`);
|
|
199
|
+
exitCode = 3;
|
|
200
|
+
break indexLogic;
|
|
298
201
|
}
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
if (
|
|
302
|
-
|
|
202
|
+
const isInteractive = process.stdout.isTTY && globalOpts.quiet !== true && globalOpts.format !== "json";
|
|
203
|
+
let spinner;
|
|
204
|
+
if (isInteractive) {
|
|
205
|
+
spinner = ora2(`Indexing store: ${store.name}`).start();
|
|
206
|
+
} else if (globalOpts.quiet !== true && globalOpts.format !== "json") {
|
|
207
|
+
console.log(`Indexing store: ${store.name}`);
|
|
208
|
+
}
|
|
209
|
+
await services.lance.initialize(store.id);
|
|
210
|
+
const result = await services.index.indexStore(store, (event) => {
|
|
211
|
+
if (event.type === "progress") {
|
|
212
|
+
if (spinner) {
|
|
213
|
+
spinner.text = `Indexing: ${String(event.current)}/${String(event.total)} files - ${event.message}`;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
});
|
|
217
|
+
if (result.success) {
|
|
218
|
+
if (globalOpts.format === "json") {
|
|
219
|
+
console.log(JSON.stringify(result.data, null, 2));
|
|
220
|
+
} else {
|
|
221
|
+
const message = `Indexed ${String(result.data.documentsIndexed)} documents, ${String(result.data.chunksCreated)} chunks in ${String(result.data.timeMs)}ms`;
|
|
222
|
+
if (spinner !== void 0) {
|
|
223
|
+
spinner.succeed(message);
|
|
224
|
+
} else if (globalOpts.quiet !== true) {
|
|
225
|
+
console.log(message);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
303
228
|
} else {
|
|
304
|
-
const message = `
|
|
229
|
+
const message = `Error: ${result.error.message}`;
|
|
305
230
|
if (spinner !== void 0) {
|
|
306
|
-
spinner.
|
|
307
|
-
} else
|
|
308
|
-
console.
|
|
231
|
+
spinner.fail(message);
|
|
232
|
+
} else {
|
|
233
|
+
console.error(message);
|
|
309
234
|
}
|
|
235
|
+
exitCode = 4;
|
|
236
|
+
break indexLogic;
|
|
310
237
|
}
|
|
311
|
-
} else {
|
|
312
|
-
const message = `Error: ${result.error.message}`;
|
|
313
|
-
if (spinner !== void 0) {
|
|
314
|
-
spinner.fail(message);
|
|
315
|
-
} else {
|
|
316
|
-
console.error(message);
|
|
317
|
-
}
|
|
318
|
-
process.exit(4);
|
|
319
238
|
}
|
|
320
239
|
} finally {
|
|
321
240
|
await destroyServices(services);
|
|
322
241
|
}
|
|
242
|
+
if (exitCode !== 0) {
|
|
243
|
+
process.exit(exitCode);
|
|
244
|
+
}
|
|
323
245
|
});
|
|
324
|
-
index.command("watch <store>").description("Watch store directory; re-index when files change").option(
|
|
246
|
+
index.command("watch <store>").description("Watch store directory; re-index when files change").option(
|
|
247
|
+
"--debounce <ms>",
|
|
248
|
+
"Wait N ms after last change before re-indexing (default: 1000)",
|
|
249
|
+
"1000"
|
|
250
|
+
).action(async (storeIdOrName, options) => {
|
|
325
251
|
const globalOpts = getOptions();
|
|
326
252
|
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
327
253
|
const store = await services.store.getByIdOrName(storeIdOrName);
|
|
@@ -329,7 +255,7 @@ function createIndexCommand(getOptions) {
|
|
|
329
255
|
console.error(`Error: File/repo store not found: ${storeIdOrName}`);
|
|
330
256
|
process.exit(3);
|
|
331
257
|
}
|
|
332
|
-
const { WatchService } = await import("./watch.service-
|
|
258
|
+
const { WatchService } = await import("./watch.service-BJV3TI3F.js");
|
|
333
259
|
const watchService = new WatchService(services.index, services.lance);
|
|
334
260
|
if (globalOpts.quiet !== true) {
|
|
335
261
|
console.log(`Watching ${store.name} for changes...`);
|
|
@@ -350,456 +276,29 @@ function createIndexCommand(getOptions) {
|
|
|
350
276
|
return index;
|
|
351
277
|
}
|
|
352
278
|
|
|
353
|
-
// src/cli/commands/
|
|
354
|
-
import { Command as
|
|
355
|
-
|
|
279
|
+
// src/cli/commands/mcp.ts
|
|
280
|
+
import { Command as Command3 } from "commander";
|
|
281
|
+
function createMCPCommand(getOptions) {
|
|
282
|
+
const mcp = new Command3("mcp").description("Start MCP (Model Context Protocol) server for AI agent integration").action(async () => {
|
|
283
|
+
const opts = getOptions();
|
|
284
|
+
await runMCPServer({
|
|
285
|
+
dataDir: opts.dataDir,
|
|
286
|
+
config: opts.config
|
|
287
|
+
});
|
|
288
|
+
});
|
|
289
|
+
return mcp;
|
|
290
|
+
}
|
|
356
291
|
|
|
357
|
-
// src/
|
|
358
|
-
import {
|
|
359
|
-
import { cors } from "hono/cors";
|
|
360
|
-
import { z } from "zod";
|
|
361
|
-
var CreateStoreBodySchema = z.object({
|
|
362
|
-
name: z.string().min(1, "Store name must be a non-empty string"),
|
|
363
|
-
type: z.enum(["file", "repo", "web"]),
|
|
364
|
-
path: z.string().min(1).optional(),
|
|
365
|
-
url: z.string().min(1).optional(),
|
|
366
|
-
description: z.string().optional(),
|
|
367
|
-
tags: z.array(z.string()).optional(),
|
|
368
|
-
branch: z.string().optional(),
|
|
369
|
-
depth: z.number().int().positive().optional()
|
|
370
|
-
}).refine(
|
|
371
|
-
(data) => {
|
|
372
|
-
switch (data.type) {
|
|
373
|
-
case "file":
|
|
374
|
-
return data.path !== void 0;
|
|
375
|
-
case "web":
|
|
376
|
-
return data.url !== void 0;
|
|
377
|
-
case "repo":
|
|
378
|
-
return data.path !== void 0 || data.url !== void 0;
|
|
379
|
-
}
|
|
380
|
-
},
|
|
381
|
-
{ message: "Missing required field: file stores need path, web stores need url, repo stores need path or url" }
|
|
382
|
-
);
|
|
383
|
-
var SearchBodySchema = z.object({
|
|
384
|
-
query: z.string().min(1, "Query must be a non-empty string"),
|
|
385
|
-
detail: z.enum(["minimal", "contextual", "full"]).optional(),
|
|
386
|
-
limit: z.number().int().positive().optional(),
|
|
387
|
-
stores: z.array(z.string()).optional()
|
|
388
|
-
});
|
|
389
|
-
function createApp(services) {
|
|
390
|
-
const app = new Hono();
|
|
391
|
-
app.use("*", cors());
|
|
392
|
-
app.get("/health", (c) => c.json({ status: "ok" }));
|
|
393
|
-
app.get("/api/stores", async (c) => {
|
|
394
|
-
const stores = await services.store.list();
|
|
395
|
-
return c.json(stores);
|
|
396
|
-
});
|
|
397
|
-
app.post("/api/stores", async (c) => {
|
|
398
|
-
const jsonData = await c.req.json();
|
|
399
|
-
const parseResult = CreateStoreBodySchema.safeParse(jsonData);
|
|
400
|
-
if (!parseResult.success) {
|
|
401
|
-
return c.json({ error: parseResult.error.issues[0]?.message ?? "Invalid request body" }, 400);
|
|
402
|
-
}
|
|
403
|
-
const result = await services.store.create(parseResult.data);
|
|
404
|
-
if (result.success) {
|
|
405
|
-
return c.json(result.data, 201);
|
|
406
|
-
}
|
|
407
|
-
return c.json({ error: result.error.message }, 400);
|
|
408
|
-
});
|
|
409
|
-
app.get("/api/stores/:id", async (c) => {
|
|
410
|
-
const store = await services.store.getByIdOrName(c.req.param("id"));
|
|
411
|
-
if (!store) return c.json({ error: "Not found" }, 404);
|
|
412
|
-
return c.json(store);
|
|
413
|
-
});
|
|
414
|
-
app.delete("/api/stores/:id", async (c) => {
|
|
415
|
-
const store = await services.store.getByIdOrName(c.req.param("id"));
|
|
416
|
-
if (!store) return c.json({ error: "Not found" }, 404);
|
|
417
|
-
const result = await services.store.delete(store.id);
|
|
418
|
-
if (result.success) return c.json({ deleted: true });
|
|
419
|
-
return c.json({ error: result.error.message }, 400);
|
|
420
|
-
});
|
|
421
|
-
app.post("/api/search", async (c) => {
|
|
422
|
-
const jsonData = await c.req.json();
|
|
423
|
-
const parseResult = SearchBodySchema.safeParse(jsonData);
|
|
424
|
-
if (!parseResult.success) {
|
|
425
|
-
return c.json({ error: parseResult.error.issues[0]?.message ?? "Invalid request body" }, 400);
|
|
426
|
-
}
|
|
427
|
-
const storeIds = (await services.store.list()).map((s) => s.id);
|
|
428
|
-
for (const id of storeIds) {
|
|
429
|
-
await services.lance.initialize(id);
|
|
430
|
-
}
|
|
431
|
-
const requestedStores = parseResult.data.stores !== void 0 ? parseResult.data.stores.map((s) => createStoreId(s)) : storeIds;
|
|
432
|
-
const query = {
|
|
433
|
-
query: parseResult.data.query,
|
|
434
|
-
detail: parseResult.data.detail ?? "minimal",
|
|
435
|
-
limit: parseResult.data.limit ?? 10,
|
|
436
|
-
stores: requestedStores
|
|
437
|
-
};
|
|
438
|
-
const results = await services.search.search(query);
|
|
439
|
-
return c.json(results);
|
|
440
|
-
});
|
|
441
|
-
app.post("/api/stores/:id/index", async (c) => {
|
|
442
|
-
const store = await services.store.getByIdOrName(c.req.param("id"));
|
|
443
|
-
if (!store) return c.json({ error: "Not found" }, 404);
|
|
444
|
-
await services.lance.initialize(store.id);
|
|
445
|
-
const result = await services.index.indexStore(store);
|
|
446
|
-
if (result.success) return c.json(result.data);
|
|
447
|
-
return c.json({ error: result.error.message }, 400);
|
|
448
|
-
});
|
|
449
|
-
return app;
|
|
450
|
-
}
|
|
451
|
-
|
|
452
|
-
// src/cli/commands/serve.ts
|
|
453
|
-
function createServeCommand(getOptions) {
|
|
454
|
-
return new Command5("serve").description("Start HTTP API server for programmatic search access").option("-p, --port <port>", "Port to listen on (default: 3847)", "3847").option("--host <host>", "Bind address (default: 127.0.0.1, use 0.0.0.0 for all interfaces)", "127.0.0.1").action(async (options) => {
|
|
455
|
-
const globalOpts = getOptions();
|
|
456
|
-
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
457
|
-
const app = createApp(services);
|
|
458
|
-
const port = parseInt(options.port ?? "3847", 10);
|
|
459
|
-
const host = options.host ?? "127.0.0.1";
|
|
460
|
-
console.log(`Starting server on http://${host}:${String(port)}`);
|
|
461
|
-
serve({
|
|
462
|
-
fetch: app.fetch,
|
|
463
|
-
port,
|
|
464
|
-
hostname: host
|
|
465
|
-
});
|
|
466
|
-
});
|
|
467
|
-
}
|
|
468
|
-
|
|
469
|
-
// src/cli/commands/crawl.ts
|
|
470
|
-
import { Command as Command6 } from "commander";
|
|
471
|
-
import { createHash } from "crypto";
|
|
472
|
-
import ora2 from "ora";
|
|
473
|
-
function createCrawlCommand(getOptions) {
|
|
474
|
-
return new Command6("crawl").description("Crawl web pages with natural language control and index into store").argument("<url>", "URL to crawl").argument("<store>", "Target web store to add crawled content to").option("--crawl <instruction>", 'Natural language instruction for what to crawl (e.g., "all Getting Started pages")').option("--extract <instruction>", 'Natural language instruction for what to extract (e.g., "extract API references")').option("--simple", "Use simple BFS mode instead of intelligent crawling").option("--max-pages <number>", "Maximum number of pages to crawl", "50").option("--headless", "Use headless browser for JavaScript-rendered sites").action(async (url, storeIdOrName, cmdOptions) => {
|
|
475
|
-
const globalOpts = getOptions();
|
|
476
|
-
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
477
|
-
let store;
|
|
478
|
-
let storeCreated = false;
|
|
479
|
-
const existingStore = await services.store.getByIdOrName(storeIdOrName);
|
|
480
|
-
if (!existingStore) {
|
|
481
|
-
const result = await services.store.create({
|
|
482
|
-
name: storeIdOrName,
|
|
483
|
-
type: "web",
|
|
484
|
-
url
|
|
485
|
-
});
|
|
486
|
-
if (!result.success) {
|
|
487
|
-
await destroyServices(services);
|
|
488
|
-
throw new Error(`Failed to create store: ${result.error.message}`);
|
|
489
|
-
}
|
|
490
|
-
const createdStore = result.data;
|
|
491
|
-
if (createdStore.type !== "web") {
|
|
492
|
-
throw new Error("Unexpected store type after creation");
|
|
493
|
-
}
|
|
494
|
-
store = createdStore;
|
|
495
|
-
storeCreated = true;
|
|
496
|
-
if (globalOpts.quiet !== true && globalOpts.format !== "json") {
|
|
497
|
-
console.log(`Created web store: ${store.name}`);
|
|
498
|
-
}
|
|
499
|
-
} else if (existingStore.type !== "web") {
|
|
500
|
-
await destroyServices(services);
|
|
501
|
-
throw new Error(`Store "${storeIdOrName}" exists but is not a web store (type: ${existingStore.type})`);
|
|
502
|
-
} else {
|
|
503
|
-
store = existingStore;
|
|
504
|
-
}
|
|
505
|
-
const maxPages = cmdOptions.maxPages !== void 0 ? parseInt(cmdOptions.maxPages) : 50;
|
|
506
|
-
const isInteractive = process.stdout.isTTY && globalOpts.quiet !== true && globalOpts.format !== "json";
|
|
507
|
-
let spinner;
|
|
508
|
-
if (isInteractive) {
|
|
509
|
-
const mode = cmdOptions.simple === true ? "simple" : "intelligent";
|
|
510
|
-
spinner = ora2(`Crawling ${url} (${mode} mode)`).start();
|
|
511
|
-
} else if (globalOpts.quiet !== true && globalOpts.format !== "json") {
|
|
512
|
-
console.log(`Crawling ${url}`);
|
|
513
|
-
}
|
|
514
|
-
const crawler = new IntelligentCrawler();
|
|
515
|
-
const webChunker = ChunkingService.forContentType("web");
|
|
516
|
-
let pagesIndexed = 0;
|
|
517
|
-
let chunksCreated = 0;
|
|
518
|
-
let exitCode = 0;
|
|
519
|
-
crawler.on("progress", (progress) => {
|
|
520
|
-
if (spinner) {
|
|
521
|
-
if (progress.type === "strategy") {
|
|
522
|
-
spinner.text = progress.message !== void 0 ? progress.message : "Analyzing crawl strategy...";
|
|
523
|
-
} else if (progress.type === "page") {
|
|
524
|
-
const url2 = progress.currentUrl !== void 0 ? progress.currentUrl : "unknown";
|
|
525
|
-
spinner.text = `Crawling ${String(progress.pagesVisited + 1)}/${String(maxPages)} - ${url2}`;
|
|
526
|
-
} else if (progress.type === "extraction") {
|
|
527
|
-
const url2 = progress.currentUrl !== void 0 ? progress.currentUrl : "unknown";
|
|
528
|
-
spinner.text = `Extracting from ${url2}...`;
|
|
529
|
-
} else if (progress.type === "error" && progress.message !== void 0) {
|
|
530
|
-
spinner.warn(progress.message);
|
|
531
|
-
}
|
|
532
|
-
}
|
|
533
|
-
});
|
|
534
|
-
try {
|
|
535
|
-
await services.lance.initialize(store.id);
|
|
536
|
-
const docs = [];
|
|
537
|
-
for await (const result of crawler.crawl(url, {
|
|
538
|
-
...cmdOptions.crawl !== void 0 && { crawlInstruction: cmdOptions.crawl },
|
|
539
|
-
...cmdOptions.extract !== void 0 && { extractInstruction: cmdOptions.extract },
|
|
540
|
-
maxPages,
|
|
541
|
-
...cmdOptions.simple !== void 0 && { simple: cmdOptions.simple },
|
|
542
|
-
useHeadless: cmdOptions.headless ?? false
|
|
543
|
-
})) {
|
|
544
|
-
const contentToProcess = result.extracted !== void 0 ? result.extracted : result.markdown;
|
|
545
|
-
const chunks = webChunker.chunk(contentToProcess, `${result.url}.md`);
|
|
546
|
-
const fileType = classifyWebContentType(result.url, result.title);
|
|
547
|
-
const urlHash = createHash("md5").update(result.url).digest("hex");
|
|
548
|
-
for (const chunk of chunks) {
|
|
549
|
-
const chunkId = chunks.length > 1 ? `${store.id}-${urlHash}-${String(chunk.chunkIndex)}` : `${store.id}-${urlHash}`;
|
|
550
|
-
const vector = await services.embeddings.embed(chunk.content);
|
|
551
|
-
docs.push({
|
|
552
|
-
id: createDocumentId(chunkId),
|
|
553
|
-
content: chunk.content,
|
|
554
|
-
vector,
|
|
555
|
-
metadata: {
|
|
556
|
-
type: chunks.length > 1 ? "chunk" : "web",
|
|
557
|
-
storeId: store.id,
|
|
558
|
-
url: result.url,
|
|
559
|
-
title: result.title,
|
|
560
|
-
extracted: result.extracted !== void 0,
|
|
561
|
-
depth: result.depth,
|
|
562
|
-
indexedAt: /* @__PURE__ */ new Date(),
|
|
563
|
-
fileType,
|
|
564
|
-
chunkIndex: chunk.chunkIndex,
|
|
565
|
-
totalChunks: chunk.totalChunks,
|
|
566
|
-
sectionHeader: chunk.sectionHeader
|
|
567
|
-
}
|
|
568
|
-
});
|
|
569
|
-
chunksCreated++;
|
|
570
|
-
}
|
|
571
|
-
pagesIndexed++;
|
|
572
|
-
}
|
|
573
|
-
if (docs.length > 0) {
|
|
574
|
-
if (spinner) {
|
|
575
|
-
spinner.text = "Indexing documents...";
|
|
576
|
-
}
|
|
577
|
-
await services.lance.addDocuments(store.id, docs);
|
|
578
|
-
}
|
|
579
|
-
const crawlResult = {
|
|
580
|
-
success: true,
|
|
581
|
-
store: store.name,
|
|
582
|
-
storeCreated,
|
|
583
|
-
url,
|
|
584
|
-
pagesCrawled: pagesIndexed,
|
|
585
|
-
chunksCreated,
|
|
586
|
-
mode: cmdOptions.simple === true ? "simple" : "intelligent",
|
|
587
|
-
hadCrawlInstruction: cmdOptions.crawl !== void 0,
|
|
588
|
-
hadExtractInstruction: cmdOptions.extract !== void 0
|
|
589
|
-
};
|
|
590
|
-
if (globalOpts.format === "json") {
|
|
591
|
-
console.log(JSON.stringify(crawlResult, null, 2));
|
|
592
|
-
} else if (spinner !== void 0) {
|
|
593
|
-
spinner.succeed(`Crawled ${String(pagesIndexed)} pages, indexed ${String(chunksCreated)} chunks`);
|
|
594
|
-
} else if (globalOpts.quiet !== true) {
|
|
595
|
-
console.log(`Crawled ${String(pagesIndexed)} pages, indexed ${String(chunksCreated)} chunks`);
|
|
596
|
-
}
|
|
597
|
-
} catch (error) {
|
|
598
|
-
const message = `Crawl failed: ${error instanceof Error ? error.message : String(error)}`;
|
|
599
|
-
if (spinner) {
|
|
600
|
-
spinner.fail(message);
|
|
601
|
-
} else {
|
|
602
|
-
console.error(`Error: ${message}`);
|
|
603
|
-
}
|
|
604
|
-
exitCode = 6;
|
|
605
|
-
} finally {
|
|
606
|
-
await crawler.stop();
|
|
607
|
-
await destroyServices(services);
|
|
608
|
-
}
|
|
609
|
-
if (exitCode !== 0) {
|
|
610
|
-
process.exit(exitCode);
|
|
611
|
-
}
|
|
612
|
-
});
|
|
613
|
-
}
|
|
292
|
+
// src/cli/commands/plugin-api.ts
|
|
293
|
+
import { Command as Command4 } from "commander";
|
|
614
294
|
|
|
615
|
-
// src/
|
|
616
|
-
import { Command as Command7 } from "commander";
|
|
617
|
-
import { spawnSync } from "child_process";
|
|
618
|
-
import { existsSync } from "fs";
|
|
619
|
-
import { mkdir } from "fs/promises";
|
|
620
|
-
import { join as join2 } from "path";
|
|
621
|
-
import { homedir } from "os";
|
|
295
|
+
// src/plugin/commands.ts
|
|
622
296
|
import ora3 from "ora";
|
|
623
297
|
|
|
624
|
-
// src/defaults/repos.ts
|
|
625
|
-
var DEFAULT_REPOS = [
|
|
626
|
-
{
|
|
627
|
-
url: "git@github.com:ericbuess/claude-code-docs.git",
|
|
628
|
-
name: "claude-code-docs",
|
|
629
|
-
description: "Claude Code documentation",
|
|
630
|
-
tags: ["claude", "docs", "claude-code"]
|
|
631
|
-
},
|
|
632
|
-
{
|
|
633
|
-
url: "git@github.com:anthropics/claude-code.git",
|
|
634
|
-
name: "claude-code",
|
|
635
|
-
description: "Claude Code CLI tool source",
|
|
636
|
-
tags: ["claude", "cli", "anthropic"]
|
|
637
|
-
},
|
|
638
|
-
{
|
|
639
|
-
url: "git@github.com:anthropics/claude-agent-sdk-python.git",
|
|
640
|
-
name: "claude-agent-sdk-python",
|
|
641
|
-
description: "Claude Agent SDK for Python",
|
|
642
|
-
tags: ["claude", "sdk", "python", "agents"]
|
|
643
|
-
},
|
|
644
|
-
{
|
|
645
|
-
url: "git@github.com:anthropics/skills.git",
|
|
646
|
-
name: "claude-skills",
|
|
647
|
-
description: "Claude skills and capabilities",
|
|
648
|
-
tags: ["claude", "skills"]
|
|
649
|
-
},
|
|
650
|
-
{
|
|
651
|
-
url: "git@github.com:anthropics/claude-quickstarts.git",
|
|
652
|
-
name: "claude-quickstarts",
|
|
653
|
-
description: "Claude quickstart examples and tutorials",
|
|
654
|
-
tags: ["claude", "examples", "tutorials"]
|
|
655
|
-
},
|
|
656
|
-
{
|
|
657
|
-
url: "git@github.com:anthropics/claude-plugins-official.git",
|
|
658
|
-
name: "claude-plugins",
|
|
659
|
-
description: "Official Claude plugins",
|
|
660
|
-
tags: ["claude", "plugins"]
|
|
661
|
-
},
|
|
662
|
-
{
|
|
663
|
-
url: "git@github.com:anthropics/claude-agent-sdk-typescript.git",
|
|
664
|
-
name: "claude-agent-sdk-typescript",
|
|
665
|
-
description: "Claude Agent SDK for TypeScript",
|
|
666
|
-
tags: ["claude", "sdk", "typescript", "agents"]
|
|
667
|
-
},
|
|
668
|
-
{
|
|
669
|
-
url: "git@github.com:anthropics/claude-agent-sdk-demos.git",
|
|
670
|
-
name: "claude-agent-sdk-demos",
|
|
671
|
-
description: "Claude Agent SDK demo applications",
|
|
672
|
-
tags: ["claude", "sdk", "demos", "examples"]
|
|
673
|
-
}
|
|
674
|
-
];
|
|
675
|
-
|
|
676
|
-
// src/cli/commands/setup.ts
|
|
677
|
-
var DEFAULT_REPOS_DIR = join2(homedir(), ".bluera", "bluera-knowledge", "repos");
|
|
678
|
-
function createSetupCommand(getOptions) {
|
|
679
|
-
const setup = new Command7("setup").description("Quick-start with pre-configured Claude/Anthropic documentation repos");
|
|
680
|
-
setup.command("repos").description("Clone repos to ~/.bluera/bluera-knowledge/repos/, create stores, index all content").option("--repos-dir <path>", "Clone destination (default: ~/.bluera/bluera-knowledge/repos/)", DEFAULT_REPOS_DIR).option("--skip-clone", "Don't clone; assume repos already exist locally").option("--skip-index", "Clone and create stores but don't index yet").option("--only <names>", "Only process matching repos (comma-separated, partial match)").option("--list", "Print available repos without cloning/indexing").action(async (options) => {
|
|
681
|
-
const globalOpts = getOptions();
|
|
682
|
-
if (options.list === true) {
|
|
683
|
-
console.log("\nDefault repositories:\n");
|
|
684
|
-
for (const repo of DEFAULT_REPOS) {
|
|
685
|
-
console.log(` ${repo.name}`);
|
|
686
|
-
console.log(` URL: ${repo.url}`);
|
|
687
|
-
console.log(` Description: ${repo.description}`);
|
|
688
|
-
console.log(` Tags: ${repo.tags.join(", ")}`);
|
|
689
|
-
console.log("");
|
|
690
|
-
}
|
|
691
|
-
return;
|
|
692
|
-
}
|
|
693
|
-
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
694
|
-
try {
|
|
695
|
-
let repos = DEFAULT_REPOS;
|
|
696
|
-
if (options.only !== void 0 && options.only !== "") {
|
|
697
|
-
const onlyNames = options.only.split(",").map((n) => n.trim().toLowerCase());
|
|
698
|
-
repos = DEFAULT_REPOS.filter(
|
|
699
|
-
(r) => onlyNames.some((n) => r.name.toLowerCase().includes(n))
|
|
700
|
-
);
|
|
701
|
-
if (repos.length === 0) {
|
|
702
|
-
console.error(`No repos matched: ${options.only}`);
|
|
703
|
-
console.log("Available repos:", DEFAULT_REPOS.map((r) => r.name).join(", "));
|
|
704
|
-
process.exit(1);
|
|
705
|
-
}
|
|
706
|
-
}
|
|
707
|
-
console.log(`
|
|
708
|
-
Setting up ${String(repos.length)} repositories...
|
|
709
|
-
`);
|
|
710
|
-
await mkdir(options.reposDir, { recursive: true });
|
|
711
|
-
for (const repo of repos) {
|
|
712
|
-
const repoPath = join2(options.reposDir, repo.name);
|
|
713
|
-
const spinner = ora3(`Processing ${repo.name}`).start();
|
|
714
|
-
try {
|
|
715
|
-
if (options.skipClone !== true) {
|
|
716
|
-
if (existsSync(repoPath)) {
|
|
717
|
-
spinner.text = `${repo.name}: Already cloned, pulling latest...`;
|
|
718
|
-
const pullResult = spawnSync("git", ["pull", "--ff-only"], { cwd: repoPath, stdio: "pipe" });
|
|
719
|
-
if (pullResult.status !== 0) {
|
|
720
|
-
spinner.text = `${repo.name}: Pull skipped (local changes)`;
|
|
721
|
-
}
|
|
722
|
-
} else {
|
|
723
|
-
spinner.text = `${repo.name}: Cloning...`;
|
|
724
|
-
const cloneResult = spawnSync("git", ["clone", repo.url, repoPath], { stdio: "pipe" });
|
|
725
|
-
if (cloneResult.status !== 0) {
|
|
726
|
-
const errorMessage = cloneResult.stderr.length > 0 ? cloneResult.stderr.toString() : "Git clone failed";
|
|
727
|
-
throw new Error(errorMessage);
|
|
728
|
-
}
|
|
729
|
-
}
|
|
730
|
-
}
|
|
731
|
-
spinner.text = `${repo.name}: Creating store...`;
|
|
732
|
-
const existingStore = await services.store.getByIdOrName(repo.name);
|
|
733
|
-
let storeId;
|
|
734
|
-
if (existingStore) {
|
|
735
|
-
storeId = existingStore.id;
|
|
736
|
-
spinner.text = `${repo.name}: Store already exists`;
|
|
737
|
-
} else {
|
|
738
|
-
const result = await services.store.create({
|
|
739
|
-
name: repo.name,
|
|
740
|
-
type: "repo",
|
|
741
|
-
path: repoPath,
|
|
742
|
-
description: repo.description,
|
|
743
|
-
tags: repo.tags
|
|
744
|
-
});
|
|
745
|
-
if (!result.success) {
|
|
746
|
-
throw new Error(result.error instanceof Error ? result.error.message : String(result.error));
|
|
747
|
-
}
|
|
748
|
-
storeId = result.data.id;
|
|
749
|
-
}
|
|
750
|
-
if (options.skipIndex !== true) {
|
|
751
|
-
spinner.text = `${repo.name}: Indexing...`;
|
|
752
|
-
const store = await services.store.getByIdOrName(storeId);
|
|
753
|
-
if (store) {
|
|
754
|
-
await services.lance.initialize(store.id);
|
|
755
|
-
const indexResult = await services.index.indexStore(store, (event) => {
|
|
756
|
-
if (event.type === "progress") {
|
|
757
|
-
spinner.text = `${repo.name}: Indexing ${String(event.current)}/${String(event.total)} files`;
|
|
758
|
-
}
|
|
759
|
-
});
|
|
760
|
-
if (indexResult.success) {
|
|
761
|
-
spinner.succeed(
|
|
762
|
-
`${repo.name}: ${String(indexResult.data.documentsIndexed)} docs, ${String(indexResult.data.chunksCreated)} chunks`
|
|
763
|
-
);
|
|
764
|
-
} else {
|
|
765
|
-
throw new Error(indexResult.error instanceof Error ? indexResult.error.message : String(indexResult.error));
|
|
766
|
-
}
|
|
767
|
-
}
|
|
768
|
-
} else {
|
|
769
|
-
spinner.succeed(`${repo.name}: Ready (indexing skipped)`);
|
|
770
|
-
}
|
|
771
|
-
} catch (error) {
|
|
772
|
-
spinner.fail(`${repo.name}: ${error instanceof Error ? error.message : String(error)}`);
|
|
773
|
-
}
|
|
774
|
-
}
|
|
775
|
-
console.log('\nSetup complete! Use "bluera-knowledge search <query>" to search.\n');
|
|
776
|
-
} finally {
|
|
777
|
-
await destroyServices(services);
|
|
778
|
-
}
|
|
779
|
-
});
|
|
780
|
-
return setup;
|
|
781
|
-
}
|
|
782
|
-
|
|
783
|
-
// src/cli/commands/mcp.ts
|
|
784
|
-
import { Command as Command8 } from "commander";
|
|
785
|
-
function createMCPCommand(getOptions) {
|
|
786
|
-
const mcp = new Command8("mcp").description("Start MCP (Model Context Protocol) server for AI agent integration").action(async () => {
|
|
787
|
-
const opts = getOptions();
|
|
788
|
-
await runMCPServer({
|
|
789
|
-
dataDir: opts.dataDir,
|
|
790
|
-
config: opts.config
|
|
791
|
-
});
|
|
792
|
-
});
|
|
793
|
-
return mcp;
|
|
794
|
-
}
|
|
795
|
-
|
|
796
|
-
// src/cli/commands/plugin-api.ts
|
|
797
|
-
import { Command as Command9 } from "commander";
|
|
798
|
-
|
|
799
298
|
// src/analysis/dependency-usage-analyzer.ts
|
|
299
|
+
import { existsSync } from "fs";
|
|
800
300
|
import { readFile, readdir } from "fs/promises";
|
|
801
|
-
import {
|
|
802
|
-
import { join as join3, extname } from "path";
|
|
301
|
+
import { join, extname } from "path";
|
|
803
302
|
var TEXT_EXTENSIONS = /* @__PURE__ */ new Set([
|
|
804
303
|
".ts",
|
|
805
304
|
".tsx",
|
|
@@ -861,25 +360,25 @@ var DependencyUsageAnalyzer = class {
|
|
|
861
360
|
if (packageName !== null && declaredDeps.has(packageName)) {
|
|
862
361
|
const dep = declaredDeps.get(packageName);
|
|
863
362
|
if (dep !== void 0) {
|
|
864
|
-
this.incrementUsage(
|
|
865
|
-
usageMap,
|
|
866
|
-
packageName,
|
|
867
|
-
filePath,
|
|
868
|
-
dep.isDev,
|
|
869
|
-
dep.language
|
|
870
|
-
);
|
|
363
|
+
this.incrementUsage(usageMap, packageName, filePath, dep.isDev, dep.language);
|
|
871
364
|
}
|
|
872
365
|
}
|
|
873
366
|
}
|
|
874
367
|
processedCount++;
|
|
875
368
|
if (onProgress !== void 0 && processedCount % 10 === 0) {
|
|
876
|
-
onProgress(
|
|
369
|
+
onProgress(
|
|
370
|
+
processedCount,
|
|
371
|
+
files.length,
|
|
372
|
+
`Analyzed ${String(processedCount)}/${String(files.length)} files`
|
|
373
|
+
);
|
|
877
374
|
}
|
|
878
375
|
} catch {
|
|
879
376
|
skippedCount++;
|
|
880
377
|
}
|
|
881
378
|
}
|
|
882
|
-
const sortedUsages = Array.from(usageMap.values()).sort(
|
|
379
|
+
const sortedUsages = Array.from(usageMap.values()).sort(
|
|
380
|
+
(a, b) => b.importCount - a.importCount
|
|
381
|
+
);
|
|
883
382
|
return ok({
|
|
884
383
|
usages: sortedUsages,
|
|
885
384
|
totalFilesScanned: processedCount,
|
|
@@ -972,9 +471,18 @@ var DependencyUsageAnalyzer = class {
|
|
|
972
471
|
try {
|
|
973
472
|
const entries = await readdir(dir, { withFileTypes: true });
|
|
974
473
|
for (const entry of entries) {
|
|
975
|
-
const fullPath =
|
|
474
|
+
const fullPath = join(dir, entry.name);
|
|
976
475
|
if (entry.isDirectory()) {
|
|
977
|
-
if (![
|
|
476
|
+
if (![
|
|
477
|
+
"node_modules",
|
|
478
|
+
".git",
|
|
479
|
+
"dist",
|
|
480
|
+
"build",
|
|
481
|
+
"coverage",
|
|
482
|
+
"__pycache__",
|
|
483
|
+
".venv",
|
|
484
|
+
"venv"
|
|
485
|
+
].includes(entry.name)) {
|
|
978
486
|
files.push(...await this.scanDirectory(fullPath));
|
|
979
487
|
}
|
|
980
488
|
} else if (entry.isFile()) {
|
|
@@ -990,8 +498,8 @@ var DependencyUsageAnalyzer = class {
|
|
|
990
498
|
}
|
|
991
499
|
async readDeclaredDependencies(projectRoot) {
|
|
992
500
|
const deps = /* @__PURE__ */ new Map();
|
|
993
|
-
const packageJsonPath =
|
|
994
|
-
if (
|
|
501
|
+
const packageJsonPath = join(projectRoot, "package.json");
|
|
502
|
+
if (existsSync(packageJsonPath)) {
|
|
995
503
|
try {
|
|
996
504
|
const content = await readFile(packageJsonPath, "utf-8");
|
|
997
505
|
const parsed = JSON.parse(content);
|
|
@@ -1010,8 +518,8 @@ var DependencyUsageAnalyzer = class {
|
|
|
1010
518
|
} catch {
|
|
1011
519
|
}
|
|
1012
520
|
}
|
|
1013
|
-
const reqPath =
|
|
1014
|
-
if (
|
|
521
|
+
const reqPath = join(projectRoot, "requirements.txt");
|
|
522
|
+
if (existsSync(reqPath)) {
|
|
1015
523
|
try {
|
|
1016
524
|
const content = await readFile(reqPath, "utf-8");
|
|
1017
525
|
const lines = content.split("\n");
|
|
@@ -1019,7 +527,7 @@ var DependencyUsageAnalyzer = class {
|
|
|
1019
527
|
const trimmed = line.trim();
|
|
1020
528
|
if (trimmed === "" || trimmed.startsWith("#")) continue;
|
|
1021
529
|
const match = /^([a-zA-Z0-9_-]+)/.exec(trimmed);
|
|
1022
|
-
if (match
|
|
530
|
+
if (match?.[1] !== void 0) {
|
|
1023
531
|
const name = match[1].toLowerCase();
|
|
1024
532
|
deps.set(name, { name, isDev: false, language: "python" });
|
|
1025
533
|
}
|
|
@@ -1027,8 +535,8 @@ var DependencyUsageAnalyzer = class {
|
|
|
1027
535
|
} catch {
|
|
1028
536
|
}
|
|
1029
537
|
}
|
|
1030
|
-
const pyprojectPath =
|
|
1031
|
-
if (
|
|
538
|
+
const pyprojectPath = join(projectRoot, "pyproject.toml");
|
|
539
|
+
if (existsSync(pyprojectPath)) {
|
|
1032
540
|
try {
|
|
1033
541
|
const content = await readFile(pyprojectPath, "utf-8");
|
|
1034
542
|
const depMatches = content.matchAll(/"([a-zA-Z0-9_-]+)"/g);
|
|
@@ -1041,13 +549,13 @@ var DependencyUsageAnalyzer = class {
|
|
|
1041
549
|
} catch {
|
|
1042
550
|
}
|
|
1043
551
|
}
|
|
1044
|
-
const cargoPath =
|
|
1045
|
-
if (
|
|
552
|
+
const cargoPath = join(projectRoot, "Cargo.toml");
|
|
553
|
+
if (existsSync(cargoPath)) {
|
|
1046
554
|
try {
|
|
1047
555
|
const content = await readFile(cargoPath, "utf-8");
|
|
1048
556
|
const inDepsSection = /\[dependencies\]([\s\S]*?)(?=\n\[|$)/;
|
|
1049
557
|
const depsMatch = inDepsSection.exec(content);
|
|
1050
|
-
if (depsMatch
|
|
558
|
+
if (depsMatch?.[1] !== void 0) {
|
|
1051
559
|
const depsSection = depsMatch[1];
|
|
1052
560
|
const cratePattern = /^([a-zA-Z0-9_-]+)\s*=/gm;
|
|
1053
561
|
for (const match of depsSection.matchAll(cratePattern)) {
|
|
@@ -1058,7 +566,7 @@ var DependencyUsageAnalyzer = class {
|
|
|
1058
566
|
}
|
|
1059
567
|
const inDevDepsSection = /\[dev-dependencies\]([\s\S]*?)(?=\n\[|$)/;
|
|
1060
568
|
const devDepsMatch = inDevDepsSection.exec(content);
|
|
1061
|
-
if (devDepsMatch
|
|
569
|
+
if (devDepsMatch?.[1] !== void 0) {
|
|
1062
570
|
const devDepsSection = devDepsMatch[1];
|
|
1063
571
|
const cratePattern = /^([a-zA-Z0-9_-]+)\s*=/gm;
|
|
1064
572
|
for (const match of devDepsSection.matchAll(cratePattern)) {
|
|
@@ -1070,8 +578,8 @@ var DependencyUsageAnalyzer = class {
|
|
|
1070
578
|
} catch {
|
|
1071
579
|
}
|
|
1072
580
|
}
|
|
1073
|
-
const goModPath =
|
|
1074
|
-
if (
|
|
581
|
+
const goModPath = join(projectRoot, "go.mod");
|
|
582
|
+
if (existsSync(goModPath)) {
|
|
1075
583
|
try {
|
|
1076
584
|
const content = await readFile(goModPath, "utf-8");
|
|
1077
585
|
const requirePattern = /^\s*([a-zA-Z0-9._/-]+)\s+v[\d.]+/gm;
|
|
@@ -1257,7 +765,6 @@ var RepoUrlResolver = class {
|
|
|
1257
765
|
};
|
|
1258
766
|
|
|
1259
767
|
// src/plugin/commands.ts
|
|
1260
|
-
import ora4 from "ora";
|
|
1261
768
|
async function handleAddRepo(args) {
|
|
1262
769
|
const services = await createServices(void 0, void 0, process.env["PWD"]);
|
|
1263
770
|
const storeName = args.name ?? extractRepoName(args.url);
|
|
@@ -1341,7 +848,7 @@ async function handleSuggest() {
|
|
|
1341
848
|
const services = await createServices(void 0, void 0, projectRoot);
|
|
1342
849
|
const analyzer = new DependencyUsageAnalyzer();
|
|
1343
850
|
const resolver = new RepoUrlResolver();
|
|
1344
|
-
const spinner =
|
|
851
|
+
const spinner = ora3("Scanning source files...").start();
|
|
1345
852
|
const result = await analyzer.analyze(projectRoot, (current, total, message) => {
|
|
1346
853
|
spinner.text = `${message} (${String(current)}/${String(total)})`;
|
|
1347
854
|
});
|
|
@@ -1351,8 +858,10 @@ async function handleSuggest() {
|
|
|
1351
858
|
process.exit(1);
|
|
1352
859
|
}
|
|
1353
860
|
const { usages, totalFilesScanned, skippedFiles } = result.data;
|
|
1354
|
-
console.log(
|
|
1355
|
-
`)
|
|
861
|
+
console.log(
|
|
862
|
+
`\u2714 Scanned ${String(totalFilesScanned)} files${skippedFiles > 0 ? ` (skipped ${String(skippedFiles)})` : ""}
|
|
863
|
+
`
|
|
864
|
+
);
|
|
1356
865
|
if (usages.length === 0) {
|
|
1357
866
|
console.log("No external dependencies found in this project.");
|
|
1358
867
|
console.log("\nMake sure you have a package.json or requirements.txt file.");
|
|
@@ -1369,23 +878,24 @@ async function handleSuggest() {
|
|
|
1369
878
|
console.log("Top dependencies by usage in this project:\n");
|
|
1370
879
|
topSuggestions.forEach((usage, i) => {
|
|
1371
880
|
console.log(`${String(i + 1)}. ${usage.packageName}`);
|
|
1372
|
-
console.log(
|
|
1373
|
-
`)
|
|
881
|
+
console.log(
|
|
882
|
+
` ${String(usage.importCount)} imports across ${String(usage.fileCount)} files
|
|
883
|
+
`
|
|
884
|
+
);
|
|
1374
885
|
});
|
|
1375
886
|
console.log("Searching for repository URLs...\n");
|
|
1376
887
|
for (const usage of topSuggestions) {
|
|
1377
|
-
const repoResult = await resolver.findRepoUrl(
|
|
1378
|
-
usage.packageName,
|
|
1379
|
-
usage.language
|
|
1380
|
-
);
|
|
888
|
+
const repoResult = await resolver.findRepoUrl(usage.packageName, usage.language);
|
|
1381
889
|
if (repoResult.url !== null) {
|
|
1382
890
|
console.log(`\u2714 ${usage.packageName}: ${repoResult.url}`);
|
|
1383
891
|
console.log(` /bluera-knowledge:add-repo ${repoResult.url} --name=${usage.packageName}
|
|
1384
892
|
`);
|
|
1385
893
|
} else {
|
|
1386
894
|
console.log(`\u2717 ${usage.packageName}: Could not find repository URL`);
|
|
1387
|
-
console.log(
|
|
1388
|
-
`
|
|
895
|
+
console.log(
|
|
896
|
+
` You can manually add it: /bluera-knowledge:add-repo <url> --name=${usage.packageName}
|
|
897
|
+
`
|
|
898
|
+
);
|
|
1389
899
|
}
|
|
1390
900
|
}
|
|
1391
901
|
console.log("Use the commands above to add repositories to your knowledge stores.");
|
|
@@ -1393,26 +903,627 @@ async function handleSuggest() {
|
|
|
1393
903
|
|
|
1394
904
|
// src/cli/commands/plugin-api.ts
|
|
1395
905
|
function createAddRepoCommand(_getOptions) {
|
|
1396
|
-
return new
|
|
906
|
+
return new Command4("add-repo").description("Clone and index a library source repository").argument("<url>", "Git repository URL").option("--name <name>", "Store name (defaults to repo name)").option("--branch <branch>", "Git branch to clone").action(async (url, options) => {
|
|
1397
907
|
await handleAddRepo({ url, ...options });
|
|
1398
908
|
});
|
|
1399
909
|
}
|
|
1400
910
|
function createAddFolderCommand(_getOptions) {
|
|
1401
|
-
return new
|
|
911
|
+
return new Command4("add-folder").description("Index a local folder of reference material").argument("<path>", "Folder path to index").option("--name <name>", "Store name (defaults to folder name)").action(async (path, options) => {
|
|
1402
912
|
await handleAddFolder({ path, ...options });
|
|
1403
913
|
});
|
|
1404
914
|
}
|
|
1405
915
|
function createStoresCommand(_getOptions) {
|
|
1406
|
-
return new
|
|
916
|
+
return new Command4("stores").description("List all indexed library stores").action(async () => {
|
|
1407
917
|
await handleStores();
|
|
1408
918
|
});
|
|
1409
919
|
}
|
|
1410
920
|
function createSuggestCommand(_getOptions) {
|
|
1411
|
-
return new
|
|
921
|
+
return new Command4("suggest").description("Suggest important dependencies to add to knowledge stores").action(async () => {
|
|
1412
922
|
await handleSuggest();
|
|
1413
923
|
});
|
|
1414
924
|
}
|
|
1415
925
|
|
|
926
|
+
// src/cli/commands/search.ts
|
|
927
|
+
import { Command as Command5 } from "commander";
|
|
928
|
+
function createSearchCommand(getOptions) {
|
|
929
|
+
const search = new Command5("search").description("Search indexed documents using vector similarity + full-text matching").argument("<query>", "Search query").option(
|
|
930
|
+
"-s, --stores <stores>",
|
|
931
|
+
"Limit search to specific stores (comma-separated IDs or names)"
|
|
932
|
+
).option(
|
|
933
|
+
"-m, --mode <mode>",
|
|
934
|
+
"vector (embeddings only), fts (text only), hybrid (both, default)",
|
|
935
|
+
"hybrid"
|
|
936
|
+
).option("-n, --limit <count>", "Maximum results to return (default: 10)", "10").option("-t, --threshold <score>", "Minimum score 0-1; omit low-relevance results").option("--include-content", "Show full document content, not just preview snippet").option(
|
|
937
|
+
"--detail <level>",
|
|
938
|
+
"Context detail: minimal, contextual, full (default: minimal)",
|
|
939
|
+
"minimal"
|
|
940
|
+
).action(
|
|
941
|
+
async (query, options) => {
|
|
942
|
+
const globalOpts = getOptions();
|
|
943
|
+
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
944
|
+
let exitCode = 0;
|
|
945
|
+
try {
|
|
946
|
+
let storeIds = (await services.store.list()).map((s) => s.id);
|
|
947
|
+
searchLogic: {
|
|
948
|
+
if (options.stores !== void 0) {
|
|
949
|
+
const requestedStores = options.stores.split(",").map((s) => s.trim());
|
|
950
|
+
const resolvedStores = [];
|
|
951
|
+
for (const requested of requestedStores) {
|
|
952
|
+
const store = await services.store.getByIdOrName(requested);
|
|
953
|
+
if (store !== void 0) {
|
|
954
|
+
resolvedStores.push(store.id);
|
|
955
|
+
} else {
|
|
956
|
+
console.error(`Error: Store not found: ${requested}`);
|
|
957
|
+
exitCode = 3;
|
|
958
|
+
break searchLogic;
|
|
959
|
+
}
|
|
960
|
+
}
|
|
961
|
+
storeIds = resolvedStores;
|
|
962
|
+
}
|
|
963
|
+
if (storeIds.length === 0) {
|
|
964
|
+
console.error("No stores to search. Create a store first.");
|
|
965
|
+
exitCode = 1;
|
|
966
|
+
break searchLogic;
|
|
967
|
+
}
|
|
968
|
+
for (const storeId of storeIds) {
|
|
969
|
+
await services.lance.initialize(storeId);
|
|
970
|
+
}
|
|
971
|
+
const results = await services.search.search({
|
|
972
|
+
query,
|
|
973
|
+
stores: storeIds,
|
|
974
|
+
mode: options.mode ?? "hybrid",
|
|
975
|
+
limit: parseInt(options.limit ?? "10", 10),
|
|
976
|
+
threshold: options.threshold !== void 0 ? parseFloat(options.threshold) : void 0,
|
|
977
|
+
includeContent: options.includeContent,
|
|
978
|
+
detail: options.detail ?? "minimal"
|
|
979
|
+
});
|
|
980
|
+
if (globalOpts.format === "json") {
|
|
981
|
+
console.log(JSON.stringify(results, null, 2));
|
|
982
|
+
} else if (globalOpts.quiet === true) {
|
|
983
|
+
for (const r of results.results) {
|
|
984
|
+
const path = r.metadata.path ?? r.metadata.url ?? "unknown";
|
|
985
|
+
console.log(path);
|
|
986
|
+
}
|
|
987
|
+
} else {
|
|
988
|
+
console.log(`
|
|
989
|
+
Search: "${query}"`);
|
|
990
|
+
console.log(
|
|
991
|
+
`Mode: ${results.mode} | Detail: ${String(options.detail)} | Stores: ${String(results.stores.length)} | Results: ${String(results.totalResults)} | Time: ${String(results.timeMs)}ms
|
|
992
|
+
`
|
|
993
|
+
);
|
|
994
|
+
if (results.results.length === 0) {
|
|
995
|
+
console.log("No results found.\n");
|
|
996
|
+
} else {
|
|
997
|
+
for (let i = 0; i < results.results.length; i++) {
|
|
998
|
+
const r = results.results[i];
|
|
999
|
+
if (r === void 0) continue;
|
|
1000
|
+
if (r.summary) {
|
|
1001
|
+
console.log(
|
|
1002
|
+
`${String(i + 1)}. [${r.score.toFixed(2)}] ${r.summary.type}: ${r.summary.name}`
|
|
1003
|
+
);
|
|
1004
|
+
console.log(` ${r.summary.location}`);
|
|
1005
|
+
console.log(` ${r.summary.purpose}`);
|
|
1006
|
+
if (r.context && options.detail !== "minimal") {
|
|
1007
|
+
console.log(` Imports: ${r.context.keyImports.slice(0, 3).join(", ")}`);
|
|
1008
|
+
console.log(
|
|
1009
|
+
` Related: ${r.context.relatedConcepts.slice(0, 3).join(", ")}`
|
|
1010
|
+
);
|
|
1011
|
+
}
|
|
1012
|
+
console.log();
|
|
1013
|
+
} else {
|
|
1014
|
+
const path = r.metadata.path ?? r.metadata.url ?? "unknown";
|
|
1015
|
+
console.log(`${String(i + 1)}. [${r.score.toFixed(2)}] ${path}`);
|
|
1016
|
+
const preview = r.highlight ?? r.content.slice(0, 150).replace(/\n/g, " ") + (r.content.length > 150 ? "..." : "");
|
|
1017
|
+
console.log(` ${preview}
|
|
1018
|
+
`);
|
|
1019
|
+
}
|
|
1020
|
+
}
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
}
|
|
1024
|
+
} finally {
|
|
1025
|
+
await destroyServices(services);
|
|
1026
|
+
}
|
|
1027
|
+
if (exitCode !== 0) {
|
|
1028
|
+
process.exit(exitCode);
|
|
1029
|
+
}
|
|
1030
|
+
}
|
|
1031
|
+
);
|
|
1032
|
+
return search;
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
// src/cli/commands/serve.ts
|
|
1036
|
+
import { serve } from "@hono/node-server";
|
|
1037
|
+
import { Command as Command6 } from "commander";
|
|
1038
|
+
|
|
1039
|
+
// src/server/app.ts
|
|
1040
|
+
import { Hono } from "hono";
|
|
1041
|
+
import { cors } from "hono/cors";
|
|
1042
|
+
import { z } from "zod";
|
|
1043
|
+
var CreateStoreBodySchema = z.object({
|
|
1044
|
+
name: z.string().min(1, "Store name must be a non-empty string"),
|
|
1045
|
+
type: z.enum(["file", "repo", "web"]),
|
|
1046
|
+
path: z.string().min(1).optional(),
|
|
1047
|
+
url: z.string().min(1).optional(),
|
|
1048
|
+
description: z.string().optional(),
|
|
1049
|
+
tags: z.array(z.string()).optional(),
|
|
1050
|
+
branch: z.string().optional(),
|
|
1051
|
+
depth: z.number().int().positive().optional()
|
|
1052
|
+
}).refine(
|
|
1053
|
+
(data) => {
|
|
1054
|
+
switch (data.type) {
|
|
1055
|
+
case "file":
|
|
1056
|
+
return data.path !== void 0;
|
|
1057
|
+
case "web":
|
|
1058
|
+
return data.url !== void 0;
|
|
1059
|
+
case "repo":
|
|
1060
|
+
return data.path !== void 0 || data.url !== void 0;
|
|
1061
|
+
}
|
|
1062
|
+
},
|
|
1063
|
+
{
|
|
1064
|
+
message: "Missing required field: file stores need path, web stores need url, repo stores need path or url"
|
|
1065
|
+
}
|
|
1066
|
+
);
|
|
1067
|
+
var SearchBodySchema = z.object({
|
|
1068
|
+
query: z.string().min(1, "Query must be a non-empty string"),
|
|
1069
|
+
detail: z.enum(["minimal", "contextual", "full"]).optional(),
|
|
1070
|
+
limit: z.number().int().positive().optional(),
|
|
1071
|
+
stores: z.array(z.string()).optional()
|
|
1072
|
+
});
|
|
1073
|
+
function createApp(services) {
|
|
1074
|
+
const app = new Hono();
|
|
1075
|
+
app.use("*", cors());
|
|
1076
|
+
app.get("/health", (c) => c.json({ status: "ok" }));
|
|
1077
|
+
app.get("/api/stores", async (c) => {
|
|
1078
|
+
const stores = await services.store.list();
|
|
1079
|
+
return c.json(stores);
|
|
1080
|
+
});
|
|
1081
|
+
app.post("/api/stores", async (c) => {
|
|
1082
|
+
const jsonData = await c.req.json();
|
|
1083
|
+
const parseResult = CreateStoreBodySchema.safeParse(jsonData);
|
|
1084
|
+
if (!parseResult.success) {
|
|
1085
|
+
return c.json({ error: parseResult.error.issues[0]?.message ?? "Invalid request body" }, 400);
|
|
1086
|
+
}
|
|
1087
|
+
const result = await services.store.create(parseResult.data);
|
|
1088
|
+
if (result.success) {
|
|
1089
|
+
return c.json(result.data, 201);
|
|
1090
|
+
}
|
|
1091
|
+
return c.json({ error: result.error.message }, 400);
|
|
1092
|
+
});
|
|
1093
|
+
app.get("/api/stores/:id", async (c) => {
|
|
1094
|
+
const store = await services.store.getByIdOrName(c.req.param("id"));
|
|
1095
|
+
if (!store) return c.json({ error: "Not found" }, 404);
|
|
1096
|
+
return c.json(store);
|
|
1097
|
+
});
|
|
1098
|
+
app.delete("/api/stores/:id", async (c) => {
|
|
1099
|
+
const store = await services.store.getByIdOrName(c.req.param("id"));
|
|
1100
|
+
if (!store) return c.json({ error: "Not found" }, 404);
|
|
1101
|
+
const result = await services.store.delete(store.id);
|
|
1102
|
+
if (result.success) return c.json({ deleted: true });
|
|
1103
|
+
return c.json({ error: result.error.message }, 400);
|
|
1104
|
+
});
|
|
1105
|
+
app.post("/api/search", async (c) => {
|
|
1106
|
+
const jsonData = await c.req.json();
|
|
1107
|
+
const parseResult = SearchBodySchema.safeParse(jsonData);
|
|
1108
|
+
if (!parseResult.success) {
|
|
1109
|
+
return c.json({ error: parseResult.error.issues[0]?.message ?? "Invalid request body" }, 400);
|
|
1110
|
+
}
|
|
1111
|
+
const storeIds = (await services.store.list()).map((s) => s.id);
|
|
1112
|
+
for (const id of storeIds) {
|
|
1113
|
+
await services.lance.initialize(id);
|
|
1114
|
+
}
|
|
1115
|
+
const requestedStores = parseResult.data.stores !== void 0 ? parseResult.data.stores.map((s) => createStoreId(s)) : storeIds;
|
|
1116
|
+
const query = {
|
|
1117
|
+
query: parseResult.data.query,
|
|
1118
|
+
detail: parseResult.data.detail ?? "minimal",
|
|
1119
|
+
limit: parseResult.data.limit ?? 10,
|
|
1120
|
+
stores: requestedStores
|
|
1121
|
+
};
|
|
1122
|
+
const results = await services.search.search(query);
|
|
1123
|
+
return c.json(results);
|
|
1124
|
+
});
|
|
1125
|
+
app.post("/api/stores/:id/index", async (c) => {
|
|
1126
|
+
const store = await services.store.getByIdOrName(c.req.param("id"));
|
|
1127
|
+
if (!store) return c.json({ error: "Not found" }, 404);
|
|
1128
|
+
await services.lance.initialize(store.id);
|
|
1129
|
+
const result = await services.index.indexStore(store);
|
|
1130
|
+
if (result.success) return c.json(result.data);
|
|
1131
|
+
return c.json({ error: result.error.message }, 400);
|
|
1132
|
+
});
|
|
1133
|
+
return app;
|
|
1134
|
+
}
|
|
1135
|
+
|
|
1136
|
+
// src/cli/commands/serve.ts
|
|
1137
|
+
function createServeCommand(getOptions) {
|
|
1138
|
+
return new Command6("serve").description("Start HTTP API server for programmatic search access").option("-p, --port <port>", "Port to listen on (default: 3847)", "3847").option(
|
|
1139
|
+
"--host <host>",
|
|
1140
|
+
"Bind address (default: 127.0.0.1, use 0.0.0.0 for all interfaces)",
|
|
1141
|
+
"127.0.0.1"
|
|
1142
|
+
).action(async (options) => {
|
|
1143
|
+
const globalOpts = getOptions();
|
|
1144
|
+
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
1145
|
+
const app = createApp(services);
|
|
1146
|
+
const port = parseInt(options.port ?? "3847", 10);
|
|
1147
|
+
const host = options.host ?? "127.0.0.1";
|
|
1148
|
+
const shutdown = () => {
|
|
1149
|
+
void (async () => {
|
|
1150
|
+
await destroyServices(services);
|
|
1151
|
+
process.exit(0);
|
|
1152
|
+
})();
|
|
1153
|
+
};
|
|
1154
|
+
process.on("SIGINT", shutdown);
|
|
1155
|
+
process.on("SIGTERM", shutdown);
|
|
1156
|
+
console.log(`Starting server on http://${host}:${String(port)}`);
|
|
1157
|
+
serve({
|
|
1158
|
+
fetch: app.fetch,
|
|
1159
|
+
port,
|
|
1160
|
+
hostname: host
|
|
1161
|
+
});
|
|
1162
|
+
});
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1165
|
+
// src/cli/commands/setup.ts
|
|
1166
|
+
import { spawnSync } from "child_process";
|
|
1167
|
+
import { existsSync as existsSync2 } from "fs";
|
|
1168
|
+
import { mkdir } from "fs/promises";
|
|
1169
|
+
import { homedir } from "os";
|
|
1170
|
+
import { join as join2 } from "path";
|
|
1171
|
+
import { Command as Command7 } from "commander";
|
|
1172
|
+
import ora4 from "ora";
|
|
1173
|
+
|
|
1174
|
+
// src/defaults/repos.ts
|
|
1175
|
+
var DEFAULT_REPOS = [
|
|
1176
|
+
{
|
|
1177
|
+
url: "git@github.com:ericbuess/claude-code-docs.git",
|
|
1178
|
+
name: "claude-code-docs",
|
|
1179
|
+
description: "Claude Code documentation",
|
|
1180
|
+
tags: ["claude", "docs", "claude-code"]
|
|
1181
|
+
},
|
|
1182
|
+
{
|
|
1183
|
+
url: "git@github.com:anthropics/claude-code.git",
|
|
1184
|
+
name: "claude-code",
|
|
1185
|
+
description: "Claude Code CLI tool source",
|
|
1186
|
+
tags: ["claude", "cli", "anthropic"]
|
|
1187
|
+
},
|
|
1188
|
+
{
|
|
1189
|
+
url: "git@github.com:anthropics/claude-agent-sdk-python.git",
|
|
1190
|
+
name: "claude-agent-sdk-python",
|
|
1191
|
+
description: "Claude Agent SDK for Python",
|
|
1192
|
+
tags: ["claude", "sdk", "python", "agents"]
|
|
1193
|
+
},
|
|
1194
|
+
{
|
|
1195
|
+
url: "git@github.com:anthropics/skills.git",
|
|
1196
|
+
name: "claude-skills",
|
|
1197
|
+
description: "Claude skills and capabilities",
|
|
1198
|
+
tags: ["claude", "skills"]
|
|
1199
|
+
},
|
|
1200
|
+
{
|
|
1201
|
+
url: "git@github.com:anthropics/claude-quickstarts.git",
|
|
1202
|
+
name: "claude-quickstarts",
|
|
1203
|
+
description: "Claude quickstart examples and tutorials",
|
|
1204
|
+
tags: ["claude", "examples", "tutorials"]
|
|
1205
|
+
},
|
|
1206
|
+
{
|
|
1207
|
+
url: "git@github.com:anthropics/claude-plugins-official.git",
|
|
1208
|
+
name: "claude-plugins",
|
|
1209
|
+
description: "Official Claude plugins",
|
|
1210
|
+
tags: ["claude", "plugins"]
|
|
1211
|
+
},
|
|
1212
|
+
{
|
|
1213
|
+
url: "git@github.com:anthropics/claude-agent-sdk-typescript.git",
|
|
1214
|
+
name: "claude-agent-sdk-typescript",
|
|
1215
|
+
description: "Claude Agent SDK for TypeScript",
|
|
1216
|
+
tags: ["claude", "sdk", "typescript", "agents"]
|
|
1217
|
+
},
|
|
1218
|
+
{
|
|
1219
|
+
url: "git@github.com:anthropics/claude-agent-sdk-demos.git",
|
|
1220
|
+
name: "claude-agent-sdk-demos",
|
|
1221
|
+
description: "Claude Agent SDK demo applications",
|
|
1222
|
+
tags: ["claude", "sdk", "demos", "examples"]
|
|
1223
|
+
}
|
|
1224
|
+
];
|
|
1225
|
+
|
|
1226
|
+
// src/cli/commands/setup.ts
|
|
1227
|
+
var DEFAULT_REPOS_DIR = join2(homedir(), ".bluera", "bluera-knowledge", "repos");
|
|
1228
|
+
function createSetupCommand(getOptions) {
|
|
1229
|
+
const setup = new Command7("setup").description(
|
|
1230
|
+
"Quick-start with pre-configured Claude/Anthropic documentation repos"
|
|
1231
|
+
);
|
|
1232
|
+
setup.command("repos").description(
|
|
1233
|
+
"Clone repos to ~/.bluera/bluera-knowledge/repos/, create stores, index all content"
|
|
1234
|
+
).option(
|
|
1235
|
+
"--repos-dir <path>",
|
|
1236
|
+
"Clone destination (default: ~/.bluera/bluera-knowledge/repos/)",
|
|
1237
|
+
DEFAULT_REPOS_DIR
|
|
1238
|
+
).option("--skip-clone", "Don't clone; assume repos already exist locally").option("--skip-index", "Clone and create stores but don't index yet").option("--only <names>", "Only process matching repos (comma-separated, partial match)").option("--list", "Print available repos without cloning/indexing").action(
|
|
1239
|
+
async (options) => {
|
|
1240
|
+
const globalOpts = getOptions();
|
|
1241
|
+
if (options.list === true) {
|
|
1242
|
+
console.log("\nDefault repositories:\n");
|
|
1243
|
+
for (const repo of DEFAULT_REPOS) {
|
|
1244
|
+
console.log(` ${repo.name}`);
|
|
1245
|
+
console.log(` URL: ${repo.url}`);
|
|
1246
|
+
console.log(` Description: ${repo.description}`);
|
|
1247
|
+
console.log(` Tags: ${repo.tags.join(", ")}`);
|
|
1248
|
+
console.log("");
|
|
1249
|
+
}
|
|
1250
|
+
return;
|
|
1251
|
+
}
|
|
1252
|
+
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
1253
|
+
try {
|
|
1254
|
+
let repos = DEFAULT_REPOS;
|
|
1255
|
+
if (options.only !== void 0 && options.only !== "") {
|
|
1256
|
+
const onlyNames = options.only.split(",").map((n) => n.trim().toLowerCase());
|
|
1257
|
+
repos = DEFAULT_REPOS.filter(
|
|
1258
|
+
(r) => onlyNames.some((n) => r.name.toLowerCase().includes(n))
|
|
1259
|
+
);
|
|
1260
|
+
if (repos.length === 0) {
|
|
1261
|
+
console.error(`No repos matched: ${options.only}`);
|
|
1262
|
+
console.log("Available repos:", DEFAULT_REPOS.map((r) => r.name).join(", "));
|
|
1263
|
+
process.exit(1);
|
|
1264
|
+
}
|
|
1265
|
+
}
|
|
1266
|
+
console.log(`
|
|
1267
|
+
Setting up ${String(repos.length)} repositories...
|
|
1268
|
+
`);
|
|
1269
|
+
await mkdir(options.reposDir, { recursive: true });
|
|
1270
|
+
for (const repo of repos) {
|
|
1271
|
+
const repoPath = join2(options.reposDir, repo.name);
|
|
1272
|
+
const spinner = ora4(`Processing ${repo.name}`).start();
|
|
1273
|
+
try {
|
|
1274
|
+
if (options.skipClone !== true) {
|
|
1275
|
+
if (existsSync2(repoPath)) {
|
|
1276
|
+
spinner.text = `${repo.name}: Already cloned, pulling latest...`;
|
|
1277
|
+
const pullResult = spawnSync("git", ["pull", "--ff-only"], {
|
|
1278
|
+
cwd: repoPath,
|
|
1279
|
+
stdio: "pipe"
|
|
1280
|
+
});
|
|
1281
|
+
if (pullResult.status !== 0) {
|
|
1282
|
+
spinner.text = `${repo.name}: Pull skipped (local changes)`;
|
|
1283
|
+
}
|
|
1284
|
+
} else {
|
|
1285
|
+
spinner.text = `${repo.name}: Cloning...`;
|
|
1286
|
+
const cloneResult = spawnSync("git", ["clone", repo.url, repoPath], {
|
|
1287
|
+
stdio: "pipe"
|
|
1288
|
+
});
|
|
1289
|
+
if (cloneResult.status !== 0) {
|
|
1290
|
+
const errorMessage = cloneResult.stderr.length > 0 ? cloneResult.stderr.toString() : "Git clone failed";
|
|
1291
|
+
throw new Error(errorMessage);
|
|
1292
|
+
}
|
|
1293
|
+
}
|
|
1294
|
+
}
|
|
1295
|
+
spinner.text = `${repo.name}: Creating store...`;
|
|
1296
|
+
const existingStore = await services.store.getByIdOrName(repo.name);
|
|
1297
|
+
let storeId;
|
|
1298
|
+
if (existingStore) {
|
|
1299
|
+
storeId = existingStore.id;
|
|
1300
|
+
spinner.text = `${repo.name}: Store already exists`;
|
|
1301
|
+
} else {
|
|
1302
|
+
const result = await services.store.create({
|
|
1303
|
+
name: repo.name,
|
|
1304
|
+
type: "repo",
|
|
1305
|
+
path: repoPath,
|
|
1306
|
+
description: repo.description,
|
|
1307
|
+
tags: repo.tags
|
|
1308
|
+
});
|
|
1309
|
+
if (!result.success) {
|
|
1310
|
+
throw new Error(
|
|
1311
|
+
result.error instanceof Error ? result.error.message : String(result.error)
|
|
1312
|
+
);
|
|
1313
|
+
}
|
|
1314
|
+
storeId = result.data.id;
|
|
1315
|
+
}
|
|
1316
|
+
if (options.skipIndex !== true) {
|
|
1317
|
+
spinner.text = `${repo.name}: Indexing...`;
|
|
1318
|
+
const store = await services.store.getByIdOrName(storeId);
|
|
1319
|
+
if (store) {
|
|
1320
|
+
await services.lance.initialize(store.id);
|
|
1321
|
+
const indexResult = await services.index.indexStore(store, (event) => {
|
|
1322
|
+
if (event.type === "progress") {
|
|
1323
|
+
spinner.text = `${repo.name}: Indexing ${String(event.current)}/${String(event.total)} files`;
|
|
1324
|
+
}
|
|
1325
|
+
});
|
|
1326
|
+
if (indexResult.success) {
|
|
1327
|
+
spinner.succeed(
|
|
1328
|
+
`${repo.name}: ${String(indexResult.data.documentsIndexed)} docs, ${String(indexResult.data.chunksCreated)} chunks`
|
|
1329
|
+
);
|
|
1330
|
+
} else {
|
|
1331
|
+
throw new Error(
|
|
1332
|
+
indexResult.error instanceof Error ? indexResult.error.message : String(indexResult.error)
|
|
1333
|
+
);
|
|
1334
|
+
}
|
|
1335
|
+
}
|
|
1336
|
+
} else {
|
|
1337
|
+
spinner.succeed(`${repo.name}: Ready (indexing skipped)`);
|
|
1338
|
+
}
|
|
1339
|
+
} catch (error) {
|
|
1340
|
+
spinner.fail(
|
|
1341
|
+
`${repo.name}: ${error instanceof Error ? error.message : String(error)}`
|
|
1342
|
+
);
|
|
1343
|
+
}
|
|
1344
|
+
}
|
|
1345
|
+
console.log('\nSetup complete! Use "bluera-knowledge search <query>" to search.\n');
|
|
1346
|
+
} finally {
|
|
1347
|
+
await destroyServices(services);
|
|
1348
|
+
}
|
|
1349
|
+
}
|
|
1350
|
+
);
|
|
1351
|
+
return setup;
|
|
1352
|
+
}
|
|
1353
|
+
|
|
1354
|
+
// src/cli/commands/store.ts
|
|
1355
|
+
import { Command as Command8 } from "commander";
|
|
1356
|
+
function createStoreCommand(getOptions) {
|
|
1357
|
+
const store = new Command8("store").description(
|
|
1358
|
+
"Manage knowledge stores (collections of indexed documents)"
|
|
1359
|
+
);
|
|
1360
|
+
store.command("list").description("Show all stores with their type (file/repo/web) and ID").option("-t, --type <type>", "Filter by type: file, repo, or web").action(async (options) => {
|
|
1361
|
+
const globalOpts = getOptions();
|
|
1362
|
+
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
1363
|
+
try {
|
|
1364
|
+
const stores = await services.store.list(options.type);
|
|
1365
|
+
if (globalOpts.format === "json") {
|
|
1366
|
+
console.log(JSON.stringify(stores, null, 2));
|
|
1367
|
+
} else if (globalOpts.quiet === true) {
|
|
1368
|
+
for (const s of stores) {
|
|
1369
|
+
console.log(s.name);
|
|
1370
|
+
}
|
|
1371
|
+
} else {
|
|
1372
|
+
if (stores.length === 0) {
|
|
1373
|
+
console.log("No stores found.");
|
|
1374
|
+
} else {
|
|
1375
|
+
console.log("\nStores:\n");
|
|
1376
|
+
for (const s of stores) {
|
|
1377
|
+
console.log(` ${s.name} (${s.type}) - ${s.id}`);
|
|
1378
|
+
}
|
|
1379
|
+
console.log("");
|
|
1380
|
+
}
|
|
1381
|
+
}
|
|
1382
|
+
} finally {
|
|
1383
|
+
await destroyServices(services);
|
|
1384
|
+
}
|
|
1385
|
+
});
|
|
1386
|
+
store.command("create <name>").description("Create a new store pointing to a local path or URL").requiredOption(
|
|
1387
|
+
"-t, --type <type>",
|
|
1388
|
+
"Store type: file (local dir), repo (git), web (crawled site)"
|
|
1389
|
+
).requiredOption("-s, --source <path>", "Local path for file/repo stores, URL for web stores").option("-d, --description <desc>", "Optional description for the store").option("--tags <tags>", "Comma-separated tags for filtering").action(
|
|
1390
|
+
async (name, options) => {
|
|
1391
|
+
const globalOpts = getOptions();
|
|
1392
|
+
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
1393
|
+
let exitCode = 0;
|
|
1394
|
+
try {
|
|
1395
|
+
const isUrl = options.source.startsWith("http://") || options.source.startsWith("https://");
|
|
1396
|
+
const result = await services.store.create({
|
|
1397
|
+
name,
|
|
1398
|
+
type: options.type,
|
|
1399
|
+
path: options.type === "file" || options.type === "repo" && !isUrl ? options.source : void 0,
|
|
1400
|
+
url: options.type === "web" || options.type === "repo" && isUrl ? options.source : void 0,
|
|
1401
|
+
description: options.description,
|
|
1402
|
+
tags: options.tags?.split(",").map((t) => t.trim())
|
|
1403
|
+
});
|
|
1404
|
+
if (result.success) {
|
|
1405
|
+
if (globalOpts.format === "json") {
|
|
1406
|
+
console.log(JSON.stringify(result.data, null, 2));
|
|
1407
|
+
} else {
|
|
1408
|
+
console.log(`
|
|
1409
|
+
Created store: ${result.data.name} (${result.data.id})
|
|
1410
|
+
`);
|
|
1411
|
+
}
|
|
1412
|
+
} else {
|
|
1413
|
+
console.error(`Error: ${result.error.message}`);
|
|
1414
|
+
exitCode = 1;
|
|
1415
|
+
}
|
|
1416
|
+
} finally {
|
|
1417
|
+
await destroyServices(services);
|
|
1418
|
+
}
|
|
1419
|
+
if (exitCode !== 0) {
|
|
1420
|
+
process.exit(exitCode);
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
);
|
|
1424
|
+
store.command("info <store>").description("Show store details: ID, type, path/URL, timestamps").action(async (storeIdOrName) => {
|
|
1425
|
+
const globalOpts = getOptions();
|
|
1426
|
+
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
1427
|
+
try {
|
|
1428
|
+
const s = await services.store.getByIdOrName(storeIdOrName);
|
|
1429
|
+
if (s === void 0) {
|
|
1430
|
+
console.error(`Error: Store not found: ${storeIdOrName}`);
|
|
1431
|
+
process.exit(3);
|
|
1432
|
+
}
|
|
1433
|
+
if (globalOpts.format === "json") {
|
|
1434
|
+
console.log(JSON.stringify(s, null, 2));
|
|
1435
|
+
} else {
|
|
1436
|
+
console.log(`
|
|
1437
|
+
Store: ${s.name}`);
|
|
1438
|
+
console.log(` ID: ${s.id}`);
|
|
1439
|
+
console.log(` Type: ${s.type}`);
|
|
1440
|
+
if ("path" in s) console.log(` Path: ${s.path}`);
|
|
1441
|
+
if ("url" in s && s.url !== void 0) console.log(` URL: ${s.url}`);
|
|
1442
|
+
if (s.description !== void 0) console.log(` Description: ${s.description}`);
|
|
1443
|
+
console.log(` Created: ${s.createdAt.toISOString()}`);
|
|
1444
|
+
console.log(` Updated: ${s.updatedAt.toISOString()}`);
|
|
1445
|
+
console.log("");
|
|
1446
|
+
}
|
|
1447
|
+
} finally {
|
|
1448
|
+
await destroyServices(services);
|
|
1449
|
+
}
|
|
1450
|
+
});
|
|
1451
|
+
store.command("delete <store>").description("Remove store and its indexed documents from LanceDB").option("-f, --force", "Delete without confirmation prompt").option("-y, --yes", "Alias for --force").action(async (storeIdOrName, options) => {
|
|
1452
|
+
const globalOpts = getOptions();
|
|
1453
|
+
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
1454
|
+
try {
|
|
1455
|
+
const s = await services.store.getByIdOrName(storeIdOrName);
|
|
1456
|
+
if (s === void 0) {
|
|
1457
|
+
console.error(`Error: Store not found: ${storeIdOrName}`);
|
|
1458
|
+
process.exit(3);
|
|
1459
|
+
}
|
|
1460
|
+
const skipConfirmation = options.force === true || options.yes === true;
|
|
1461
|
+
if (!skipConfirmation) {
|
|
1462
|
+
if (!process.stdin.isTTY) {
|
|
1463
|
+
console.error(
|
|
1464
|
+
"Error: Use --force or -y to delete without confirmation in non-interactive mode"
|
|
1465
|
+
);
|
|
1466
|
+
process.exit(1);
|
|
1467
|
+
}
|
|
1468
|
+
const readline = await import("readline");
|
|
1469
|
+
const rl = readline.createInterface({
|
|
1470
|
+
input: process.stdin,
|
|
1471
|
+
output: process.stdout
|
|
1472
|
+
});
|
|
1473
|
+
const answer = await new Promise((resolve) => {
|
|
1474
|
+
rl.question(`Delete store "${s.name}"? [y/N] `, resolve);
|
|
1475
|
+
});
|
|
1476
|
+
rl.close();
|
|
1477
|
+
if (answer.toLowerCase() !== "y" && answer.toLowerCase() !== "yes") {
|
|
1478
|
+
console.log("Cancelled.");
|
|
1479
|
+
process.exit(0);
|
|
1480
|
+
}
|
|
1481
|
+
}
|
|
1482
|
+
const result = await services.store.delete(s.id);
|
|
1483
|
+
if (result.success) {
|
|
1484
|
+
console.log(`Deleted store: ${s.name}`);
|
|
1485
|
+
} else {
|
|
1486
|
+
console.error(`Error: ${result.error.message}`);
|
|
1487
|
+
process.exit(1);
|
|
1488
|
+
}
|
|
1489
|
+
} finally {
|
|
1490
|
+
await destroyServices(services);
|
|
1491
|
+
}
|
|
1492
|
+
});
|
|
1493
|
+
return store;
|
|
1494
|
+
}
|
|
1495
|
+
|
|
1496
|
+
// src/cli/program.ts
|
|
1497
|
+
import { readFileSync } from "fs";
|
|
1498
|
+
import { dirname, join as join3 } from "path";
|
|
1499
|
+
import { fileURLToPath } from "url";
|
|
1500
|
+
import { Command as Command9 } from "commander";
|
|
1501
|
+
function getVersion() {
|
|
1502
|
+
const __filename2 = fileURLToPath(import.meta.url);
|
|
1503
|
+
const __dirname2 = dirname(__filename2);
|
|
1504
|
+
const content = readFileSync(join3(__dirname2, "../package.json"), "utf-8");
|
|
1505
|
+
const pkg = JSON.parse(content);
|
|
1506
|
+
return pkg.version;
|
|
1507
|
+
}
|
|
1508
|
+
var version = getVersion();
|
|
1509
|
+
function createProgram() {
|
|
1510
|
+
const program2 = new Command9();
|
|
1511
|
+
program2.name("bluera-knowledge").description("CLI tool for managing knowledge stores with semantic search").version(version);
|
|
1512
|
+
program2.option("-c, --config <path>", "Path to config file").option("-d, --data-dir <path>", "Data directory").option("-p, --project-root <path>", "Project root directory (for resolving relative paths)").option("-f, --format <format>", "Output format: json | table | plain", "table").option("-q, --quiet", "Suppress non-essential output").option("-v, --verbose", "Enable verbose logging");
|
|
1513
|
+
return program2;
|
|
1514
|
+
}
|
|
1515
|
+
function getGlobalOptions(program2) {
|
|
1516
|
+
const opts = program2.opts();
|
|
1517
|
+
return {
|
|
1518
|
+
config: opts.config,
|
|
1519
|
+
dataDir: opts.dataDir,
|
|
1520
|
+
projectRoot: opts.projectRoot,
|
|
1521
|
+
format: opts.format,
|
|
1522
|
+
quiet: opts.quiet,
|
|
1523
|
+
verbose: opts.verbose
|
|
1524
|
+
};
|
|
1525
|
+
}
|
|
1526
|
+
|
|
1416
1527
|
// src/index.ts
|
|
1417
1528
|
var DEFAULT_DATA_DIR = join4(homedir2(), ".bluera", "bluera-knowledge", "data");
|
|
1418
1529
|
var DEFAULT_CONFIG = join4(homedir2(), ".bluera", "bluera-knowledge", "config.json");
|
|
@@ -1425,7 +1536,7 @@ function formatCommandHelp(cmd, indent = "") {
|
|
|
1425
1536
|
const req = a.required;
|
|
1426
1537
|
return req ? `<${a.name()}>` : `[${a.name()}]`;
|
|
1427
1538
|
}).join(" ");
|
|
1428
|
-
lines.push(`${indent}${name}${args ?
|
|
1539
|
+
lines.push(`${indent}${name}${args ? ` ${args}` : ""}`);
|
|
1429
1540
|
if (desc) {
|
|
1430
1541
|
lines.push(`${indent} ${desc}`);
|
|
1431
1542
|
}
|
|
@@ -1436,7 +1547,7 @@ function formatCommandHelp(cmd, indent = "") {
|
|
|
1436
1547
|
const subcommands = cmd.commands.filter((c) => c.name() !== "help");
|
|
1437
1548
|
for (const sub of subcommands) {
|
|
1438
1549
|
lines.push("");
|
|
1439
|
-
lines.push(...formatCommandHelp(sub, indent
|
|
1550
|
+
lines.push(...formatCommandHelp(sub, `${indent} `));
|
|
1440
1551
|
}
|
|
1441
1552
|
return lines;
|
|
1442
1553
|
}
|
|
@@ -1447,7 +1558,9 @@ function printFullHelp(program2) {
|
|
|
1447
1558
|
console.log(` config ${DEFAULT_CONFIG}`);
|
|
1448
1559
|
console.log(` repos ${DEFAULT_REPOS_DIR2}`);
|
|
1449
1560
|
console.log("\nGlobal options:");
|
|
1450
|
-
const globalOpts = program2.options.filter(
|
|
1561
|
+
const globalOpts = program2.options.filter(
|
|
1562
|
+
(o) => o.flags !== "-h, --help" && o.flags !== "-V, --version"
|
|
1563
|
+
);
|
|
1451
1564
|
for (const opt of globalOpts) {
|
|
1452
1565
|
console.log(` ${opt.flags.padEnd(28)} ${opt.description}`);
|
|
1453
1566
|
}
|