bluera-knowledge 0.9.31 → 0.9.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/commands/code-review.md +15 -0
- package/.claude/hooks/post-edit-check.sh +5 -3
- package/.claude/skills/atomic-commits/SKILL.md +3 -1
- package/.claude/skills/code-review-repo/skill.md +62 -0
- package/.husky/pre-commit +3 -2
- package/.prettierrc +9 -0
- package/.versionrc.json +1 -1
- package/CHANGELOG.md +35 -0
- package/CLAUDE.md +6 -0
- package/README.md +25 -13
- package/bun.lock +277 -33
- package/dist/{chunk-L2YVNC63.js → chunk-6FHWC36B.js} +9 -1
- package/dist/chunk-6FHWC36B.js.map +1 -0
- package/dist/{chunk-2SJHNRXD.js → chunk-DC7CGSGT.js} +288 -241
- package/dist/chunk-DC7CGSGT.js.map +1 -0
- package/dist/{chunk-RWSXP3PQ.js → chunk-WFNPNAAP.js} +3194 -3024
- package/dist/chunk-WFNPNAAP.js.map +1 -0
- package/dist/{chunk-OGEY66FZ.js → chunk-Z2KKVH45.js} +548 -482
- package/dist/chunk-Z2KKVH45.js.map +1 -0
- package/dist/index.js +871 -754
- package/dist/index.js.map +1 -1
- package/dist/mcp/server.js +3 -3
- package/dist/watch.service-BJV3TI3F.js +7 -0
- package/dist/workers/background-worker-cli.js +46 -45
- package/dist/workers/background-worker-cli.js.map +1 -1
- package/eslint.config.js +43 -1
- package/package.json +18 -11
- package/plugin.json +8 -0
- package/python/requirements.txt +1 -1
- package/src/analysis/ast-parser.test.ts +12 -11
- package/src/analysis/ast-parser.ts +28 -22
- package/src/analysis/code-graph.test.ts +52 -62
- package/src/analysis/code-graph.ts +9 -13
- package/src/analysis/dependency-usage-analyzer.test.ts +91 -271
- package/src/analysis/dependency-usage-analyzer.ts +52 -24
- package/src/analysis/go-ast-parser.test.ts +22 -22
- package/src/analysis/go-ast-parser.ts +18 -25
- package/src/analysis/parser-factory.test.ts +9 -9
- package/src/analysis/parser-factory.ts +3 -3
- package/src/analysis/python-ast-parser.test.ts +27 -27
- package/src/analysis/python-ast-parser.ts +2 -2
- package/src/analysis/repo-url-resolver.test.ts +82 -82
- package/src/analysis/rust-ast-parser.test.ts +19 -19
- package/src/analysis/rust-ast-parser.ts +17 -27
- package/src/analysis/tree-sitter-parser.test.ts +3 -3
- package/src/analysis/tree-sitter-parser.ts +10 -16
- package/src/cli/commands/crawl.test.ts +40 -24
- package/src/cli/commands/crawl.ts +186 -161
- package/src/cli/commands/index-cmd.test.ts +90 -90
- package/src/cli/commands/index-cmd.ts +52 -36
- package/src/cli/commands/mcp.test.ts +6 -6
- package/src/cli/commands/mcp.ts +2 -2
- package/src/cli/commands/plugin-api.test.ts +16 -18
- package/src/cli/commands/plugin-api.ts +9 -6
- package/src/cli/commands/search.test.ts +16 -7
- package/src/cli/commands/search.ts +124 -87
- package/src/cli/commands/serve.test.ts +67 -25
- package/src/cli/commands/serve.ts +18 -3
- package/src/cli/commands/setup.test.ts +176 -101
- package/src/cli/commands/setup.ts +140 -117
- package/src/cli/commands/store.test.ts +82 -53
- package/src/cli/commands/store.ts +56 -37
- package/src/cli/program.ts +2 -2
- package/src/crawl/article-converter.test.ts +4 -1
- package/src/crawl/article-converter.ts +46 -31
- package/src/crawl/bridge.test.ts +240 -132
- package/src/crawl/bridge.ts +87 -30
- package/src/crawl/claude-client.test.ts +124 -56
- package/src/crawl/claude-client.ts +7 -15
- package/src/crawl/intelligent-crawler.test.ts +65 -22
- package/src/crawl/intelligent-crawler.ts +86 -53
- package/src/crawl/markdown-utils.ts +1 -4
- package/src/db/embeddings.ts +4 -6
- package/src/db/lance.test.ts +63 -4
- package/src/db/lance.ts +31 -12
- package/src/index.ts +26 -17
- package/src/logging/index.ts +1 -5
- package/src/logging/logger.ts +3 -5
- package/src/logging/payload.test.ts +1 -1
- package/src/logging/payload.ts +3 -5
- package/src/mcp/commands/index.ts +2 -2
- package/src/mcp/commands/job.commands.ts +12 -18
- package/src/mcp/commands/meta.commands.ts +13 -13
- package/src/mcp/commands/registry.ts +5 -8
- package/src/mcp/commands/store.commands.ts +19 -19
- package/src/mcp/handlers/execute.handler.test.ts +10 -10
- package/src/mcp/handlers/execute.handler.ts +4 -5
- package/src/mcp/handlers/index.ts +10 -14
- package/src/mcp/handlers/job.handler.test.ts +10 -10
- package/src/mcp/handlers/job.handler.ts +22 -25
- package/src/mcp/handlers/search.handler.test.ts +36 -65
- package/src/mcp/handlers/search.handler.ts +135 -104
- package/src/mcp/handlers/store.handler.test.ts +41 -52
- package/src/mcp/handlers/store.handler.ts +108 -88
- package/src/mcp/schemas/index.test.ts +73 -68
- package/src/mcp/schemas/index.ts +18 -12
- package/src/mcp/server.test.ts +1 -1
- package/src/mcp/server.ts +59 -46
- package/src/plugin/commands.test.ts +230 -95
- package/src/plugin/commands.ts +24 -25
- package/src/plugin/dependency-analyzer.test.ts +52 -52
- package/src/plugin/dependency-analyzer.ts +85 -22
- package/src/plugin/git-clone.test.ts +24 -13
- package/src/plugin/git-clone.ts +3 -7
- package/src/server/app.test.ts +109 -109
- package/src/server/app.ts +32 -23
- package/src/server/index.test.ts +64 -66
- package/src/services/chunking.service.test.ts +32 -32
- package/src/services/chunking.service.ts +16 -9
- package/src/services/code-graph.service.test.ts +30 -36
- package/src/services/code-graph.service.ts +24 -10
- package/src/services/code-unit.service.test.ts +55 -11
- package/src/services/code-unit.service.ts +85 -11
- package/src/services/config.service.test.ts +37 -18
- package/src/services/config.service.ts +30 -7
- package/src/services/index.service.test.ts +49 -18
- package/src/services/index.service.ts +98 -48
- package/src/services/index.ts +8 -10
- package/src/services/job.service.test.ts +22 -22
- package/src/services/job.service.ts +18 -18
- package/src/services/project-root.service.test.ts +1 -3
- package/src/services/search.service.test.ts +248 -120
- package/src/services/search.service.ts +286 -156
- package/src/services/services.test.ts +36 -0
- package/src/services/snippet.service.test.ts +14 -6
- package/src/services/snippet.service.ts +7 -5
- package/src/services/store.service.test.ts +68 -29
- package/src/services/store.service.ts +41 -12
- package/src/services/watch.service.test.ts +34 -14
- package/src/services/watch.service.ts +11 -1
- package/src/types/brands.test.ts +3 -1
- package/src/types/index.ts +2 -13
- package/src/types/search.ts +10 -8
- package/src/utils/type-guards.test.ts +20 -15
- package/src/utils/type-guards.ts +1 -1
- package/src/workers/background-worker-cli.ts +2 -2
- package/src/workers/background-worker.test.ts +54 -40
- package/src/workers/background-worker.ts +76 -60
- package/src/workers/spawn-worker.test.ts +22 -10
- package/src/workers/spawn-worker.ts +6 -6
- package/tests/analysis/ast-parser.test.ts +3 -3
- package/tests/analysis/code-graph.test.ts +5 -5
- package/tests/fixtures/code-snippets/api/error-handling.ts +4 -15
- package/tests/fixtures/code-snippets/api/rest-controller.ts +3 -9
- package/tests/fixtures/code-snippets/auth/jwt-auth.ts +5 -21
- package/tests/fixtures/code-snippets/auth/oauth-flow.ts +4 -4
- package/tests/fixtures/code-snippets/database/repository-pattern.ts +11 -3
- package/tests/fixtures/corpus/oss-repos/hono/src/adapter/aws-lambda/handler.ts +2 -2
- package/tests/fixtures/corpus/oss-repos/hono/src/adapter/cloudflare-pages/handler.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/adapter/cloudflare-workers/serve-static.ts +2 -2
- package/tests/fixtures/corpus/oss-repos/hono/src/client/client.ts +2 -2
- package/tests/fixtures/corpus/oss-repos/hono/src/client/types.ts +22 -20
- package/tests/fixtures/corpus/oss-repos/hono/src/context.ts +13 -10
- package/tests/fixtures/corpus/oss-repos/hono/src/helper/accepts/accepts.ts +10 -7
- package/tests/fixtures/corpus/oss-repos/hono/src/helper/adapter/index.ts +2 -2
- package/tests/fixtures/corpus/oss-repos/hono/src/helper/css/index.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/helper/factory/index.ts +16 -16
- package/tests/fixtures/corpus/oss-repos/hono/src/helper/ssg/ssg.ts +2 -2
- package/tests/fixtures/corpus/oss-repos/hono/src/hono-base.ts +3 -3
- package/tests/fixtures/corpus/oss-repos/hono/src/hono.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/jsx/dom/css.ts +2 -2
- package/tests/fixtures/corpus/oss-repos/hono/src/jsx/dom/intrinsic-element/components.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/jsx/dom/render.ts +7 -7
- package/tests/fixtures/corpus/oss-repos/hono/src/jsx/hooks/index.ts +3 -3
- package/tests/fixtures/corpus/oss-repos/hono/src/jsx/intrinsic-element/components.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/jsx/utils.ts +6 -6
- package/tests/fixtures/corpus/oss-repos/hono/src/middleware/jsx-renderer/index.ts +3 -3
- package/tests/fixtures/corpus/oss-repos/hono/src/middleware/serve-static/index.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/preset/quick.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/preset/tiny.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/router/pattern-router/router.ts +2 -2
- package/tests/fixtures/corpus/oss-repos/hono/src/router/reg-exp-router/node.ts +4 -4
- package/tests/fixtures/corpus/oss-repos/hono/src/router/reg-exp-router/router.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/router/trie-router/node.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/types.ts +166 -169
- package/tests/fixtures/corpus/oss-repos/hono/src/utils/body.ts +8 -8
- package/tests/fixtures/corpus/oss-repos/hono/src/utils/color.ts +3 -3
- package/tests/fixtures/corpus/oss-repos/hono/src/utils/cookie.ts +2 -2
- package/tests/fixtures/corpus/oss-repos/hono/src/utils/encode.ts +2 -2
- package/tests/fixtures/corpus/oss-repos/hono/src/utils/types.ts +30 -33
- package/tests/fixtures/corpus/oss-repos/hono/src/validator/validator.ts +2 -2
- package/tests/fixtures/test-server.ts +3 -2
- package/tests/helpers/performance-metrics.ts +8 -25
- package/tests/helpers/search-relevance.ts +14 -69
- package/tests/integration/cli-consistency.test.ts +5 -4
- package/tests/integration/e2e-workflow.test.ts +2 -0
- package/tests/integration/python-bridge.test.ts +13 -3
- package/tests/mcp/server.test.ts +1 -1
- package/tests/services/code-unit.service.test.ts +48 -0
- package/tests/services/job.service.test.ts +124 -0
- package/tests/services/search.progressive-context.test.ts +2 -2
- package/.claude-plugin/plugin.json +0 -13
- package/BUGS-FOUND.md +0 -71
- package/dist/chunk-2SJHNRXD.js.map +0 -1
- package/dist/chunk-L2YVNC63.js.map +0 -1
- package/dist/chunk-OGEY66FZ.js.map +0 -1
- package/dist/chunk-RWSXP3PQ.js.map +0 -1
- package/dist/watch.service-YAIKKDCF.js +0 -7
- package/skills/atomic-commits/SKILL.md +0 -77
- /package/dist/{watch.service-YAIKKDCF.js.map → watch.service-BJV3TI3F.js.map} +0 -0
package/dist/index.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
runMCPServer
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-Z2KKVH45.js";
|
|
5
5
|
import {
|
|
6
6
|
IntelligentCrawler
|
|
7
|
-
} from "./chunk-
|
|
7
|
+
} from "./chunk-DC7CGSGT.js";
|
|
8
8
|
import {
|
|
9
9
|
ASTParser,
|
|
10
10
|
ChunkingService,
|
|
@@ -16,312 +16,238 @@ import {
|
|
|
16
16
|
err,
|
|
17
17
|
extractRepoName,
|
|
18
18
|
ok
|
|
19
|
-
} from "./chunk-
|
|
20
|
-
import "./chunk-
|
|
19
|
+
} from "./chunk-WFNPNAAP.js";
|
|
20
|
+
import "./chunk-6FHWC36B.js";
|
|
21
21
|
|
|
22
22
|
// src/index.ts
|
|
23
23
|
import { homedir as homedir2 } from "os";
|
|
24
24
|
import { join as join4 } from "path";
|
|
25
25
|
|
|
26
|
-
// src/cli/
|
|
26
|
+
// src/cli/commands/crawl.ts
|
|
27
|
+
import { createHash } from "crypto";
|
|
27
28
|
import { Command } from "commander";
|
|
28
|
-
import
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
quiet: opts.quiet,
|
|
53
|
-
verbose: opts.verbose
|
|
54
|
-
};
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
// src/cli/commands/store.ts
|
|
58
|
-
import { Command as Command2 } from "commander";
|
|
59
|
-
function createStoreCommand(getOptions) {
|
|
60
|
-
const store = new Command2("store").description("Manage knowledge stores (collections of indexed documents)");
|
|
61
|
-
store.command("list").description("Show all stores with their type (file/repo/web) and ID").option("-t, --type <type>", "Filter by type: file, repo, or web").action(async (options) => {
|
|
62
|
-
const globalOpts = getOptions();
|
|
63
|
-
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
64
|
-
try {
|
|
65
|
-
const stores = await services.store.list(options.type);
|
|
66
|
-
if (globalOpts.format === "json") {
|
|
67
|
-
console.log(JSON.stringify(stores, null, 2));
|
|
68
|
-
} else if (globalOpts.quiet === true) {
|
|
69
|
-
for (const s of stores) {
|
|
70
|
-
console.log(s.name);
|
|
29
|
+
import ora from "ora";
|
|
30
|
+
function createCrawlCommand(getOptions) {
|
|
31
|
+
return new Command("crawl").description("Crawl web pages with natural language control and index into store").argument("<url>", "URL to crawl").argument("<store>", "Target web store to add crawled content to").option(
|
|
32
|
+
"--crawl <instruction>",
|
|
33
|
+
'Natural language instruction for what to crawl (e.g., "all Getting Started pages")'
|
|
34
|
+
).option(
|
|
35
|
+
"--extract <instruction>",
|
|
36
|
+
'Natural language instruction for what to extract (e.g., "extract API references")'
|
|
37
|
+
).option("--simple", "Use simple BFS mode instead of intelligent crawling").option("--max-pages <number>", "Maximum number of pages to crawl", "50").option("--headless", "Use headless browser for JavaScript-rendered sites").action(
|
|
38
|
+
async (url, storeIdOrName, cmdOptions) => {
|
|
39
|
+
const globalOpts = getOptions();
|
|
40
|
+
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
41
|
+
let store;
|
|
42
|
+
let storeCreated = false;
|
|
43
|
+
const existingStore = await services.store.getByIdOrName(storeIdOrName);
|
|
44
|
+
if (!existingStore) {
|
|
45
|
+
const result = await services.store.create({
|
|
46
|
+
name: storeIdOrName,
|
|
47
|
+
type: "web",
|
|
48
|
+
url
|
|
49
|
+
});
|
|
50
|
+
if (!result.success) {
|
|
51
|
+
await destroyServices(services);
|
|
52
|
+
throw new Error(`Failed to create store: ${result.error.message}`);
|
|
71
53
|
}
|
|
72
|
-
|
|
73
|
-
if (
|
|
74
|
-
|
|
75
|
-
} else {
|
|
76
|
-
console.log("\nStores:\n");
|
|
77
|
-
for (const s of stores) {
|
|
78
|
-
console.log(` ${s.name} (${s.type}) - ${s.id}`);
|
|
79
|
-
}
|
|
80
|
-
console.log("");
|
|
54
|
+
const createdStore = result.data;
|
|
55
|
+
if (createdStore.type !== "web") {
|
|
56
|
+
throw new Error("Unexpected store type after creation");
|
|
81
57
|
}
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
});
|
|
87
|
-
store.command("create <name>").description("Create a new store pointing to a local path or URL").requiredOption("-t, --type <type>", "Store type: file (local dir), repo (git), web (crawled site)").requiredOption("-s, --source <path>", "Local path for file/repo stores, URL for web stores").option("-d, --description <desc>", "Optional description for the store").option("--tags <tags>", "Comma-separated tags for filtering").action(async (name, options) => {
|
|
88
|
-
const globalOpts = getOptions();
|
|
89
|
-
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
90
|
-
let exitCode = 0;
|
|
91
|
-
try {
|
|
92
|
-
const isUrl = options.source.startsWith("http://") || options.source.startsWith("https://");
|
|
93
|
-
const result = await services.store.create({
|
|
94
|
-
name,
|
|
95
|
-
type: options.type,
|
|
96
|
-
path: options.type === "file" || options.type === "repo" && !isUrl ? options.source : void 0,
|
|
97
|
-
url: options.type === "web" || options.type === "repo" && isUrl ? options.source : void 0,
|
|
98
|
-
description: options.description,
|
|
99
|
-
tags: options.tags?.split(",").map((t) => t.trim())
|
|
100
|
-
});
|
|
101
|
-
if (result.success) {
|
|
102
|
-
if (globalOpts.format === "json") {
|
|
103
|
-
console.log(JSON.stringify(result.data, null, 2));
|
|
104
|
-
} else {
|
|
105
|
-
console.log(`
|
|
106
|
-
Created store: ${result.data.name} (${result.data.id})
|
|
107
|
-
`);
|
|
58
|
+
store = createdStore;
|
|
59
|
+
storeCreated = true;
|
|
60
|
+
if (globalOpts.quiet !== true && globalOpts.format !== "json") {
|
|
61
|
+
console.log(`Created web store: ${store.name}`);
|
|
108
62
|
}
|
|
63
|
+
} else if (existingStore.type !== "web") {
|
|
64
|
+
await destroyServices(services);
|
|
65
|
+
throw new Error(
|
|
66
|
+
`Store "${storeIdOrName}" exists but is not a web store (type: ${existingStore.type})`
|
|
67
|
+
);
|
|
109
68
|
} else {
|
|
110
|
-
|
|
111
|
-
exitCode = 1;
|
|
112
|
-
}
|
|
113
|
-
} finally {
|
|
114
|
-
await destroyServices(services);
|
|
115
|
-
}
|
|
116
|
-
if (exitCode !== 0) {
|
|
117
|
-
process.exit(exitCode);
|
|
118
|
-
}
|
|
119
|
-
});
|
|
120
|
-
store.command("info <store>").description("Show store details: ID, type, path/URL, timestamps").action(async (storeIdOrName) => {
|
|
121
|
-
const globalOpts = getOptions();
|
|
122
|
-
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
123
|
-
try {
|
|
124
|
-
const s = await services.store.getByIdOrName(storeIdOrName);
|
|
125
|
-
if (s === void 0) {
|
|
126
|
-
console.error(`Error: Store not found: ${storeIdOrName}`);
|
|
127
|
-
process.exit(3);
|
|
128
|
-
}
|
|
129
|
-
if (globalOpts.format === "json") {
|
|
130
|
-
console.log(JSON.stringify(s, null, 2));
|
|
131
|
-
} else {
|
|
132
|
-
console.log(`
|
|
133
|
-
Store: ${s.name}`);
|
|
134
|
-
console.log(` ID: ${s.id}`);
|
|
135
|
-
console.log(` Type: ${s.type}`);
|
|
136
|
-
if ("path" in s) console.log(` Path: ${s.path}`);
|
|
137
|
-
if ("url" in s && s.url !== void 0) console.log(` URL: ${s.url}`);
|
|
138
|
-
if (s.description !== void 0) console.log(` Description: ${s.description}`);
|
|
139
|
-
console.log(` Created: ${s.createdAt.toISOString()}`);
|
|
140
|
-
console.log(` Updated: ${s.updatedAt.toISOString()}`);
|
|
141
|
-
console.log("");
|
|
69
|
+
store = existingStore;
|
|
142
70
|
}
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
const s = await services.store.getByIdOrName(storeIdOrName);
|
|
152
|
-
if (s === void 0) {
|
|
153
|
-
console.error(`Error: Store not found: ${storeIdOrName}`);
|
|
154
|
-
process.exit(3);
|
|
71
|
+
const maxPages = cmdOptions.maxPages !== void 0 ? parseInt(cmdOptions.maxPages) : 50;
|
|
72
|
+
const isInteractive = process.stdout.isTTY && globalOpts.quiet !== true && globalOpts.format !== "json";
|
|
73
|
+
let spinner;
|
|
74
|
+
if (isInteractive) {
|
|
75
|
+
const mode = cmdOptions.simple === true ? "simple" : "intelligent";
|
|
76
|
+
spinner = ora(`Crawling ${url} (${mode} mode)`).start();
|
|
77
|
+
} else if (globalOpts.quiet !== true && globalOpts.format !== "json") {
|
|
78
|
+
console.log(`Crawling ${url}`);
|
|
155
79
|
}
|
|
156
|
-
const
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
80
|
+
const crawler = new IntelligentCrawler();
|
|
81
|
+
const webChunker = ChunkingService.forContentType("web");
|
|
82
|
+
let pagesIndexed = 0;
|
|
83
|
+
let chunksCreated = 0;
|
|
84
|
+
let exitCode = 0;
|
|
85
|
+
crawler.on("progress", (progress) => {
|
|
86
|
+
if (spinner) {
|
|
87
|
+
if (progress.type === "strategy") {
|
|
88
|
+
spinner.text = progress.message ?? "Analyzing crawl strategy...";
|
|
89
|
+
} else if (progress.type === "page") {
|
|
90
|
+
const url2 = progress.currentUrl ?? "unknown";
|
|
91
|
+
spinner.text = `Crawling ${String(progress.pagesVisited + 1)}/${String(maxPages)} - ${url2}`;
|
|
92
|
+
} else if (progress.type === "extraction") {
|
|
93
|
+
const url2 = progress.currentUrl ?? "unknown";
|
|
94
|
+
spinner.text = `Extracting from ${url2}...`;
|
|
95
|
+
} else if (progress.type === "error" && progress.message !== void 0) {
|
|
96
|
+
spinner.warn(progress.message);
|
|
97
|
+
}
|
|
161
98
|
}
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
99
|
+
});
|
|
100
|
+
try {
|
|
101
|
+
await services.lance.initialize(store.id);
|
|
102
|
+
const docs = [];
|
|
103
|
+
for await (const result of crawler.crawl(url, {
|
|
104
|
+
...cmdOptions.crawl !== void 0 && { crawlInstruction: cmdOptions.crawl },
|
|
105
|
+
...cmdOptions.extract !== void 0 && { extractInstruction: cmdOptions.extract },
|
|
106
|
+
maxPages,
|
|
107
|
+
...cmdOptions.simple !== void 0 && { simple: cmdOptions.simple },
|
|
108
|
+
useHeadless: cmdOptions.headless ?? false
|
|
109
|
+
})) {
|
|
110
|
+
const contentToProcess = result.extracted ?? result.markdown;
|
|
111
|
+
const chunks = webChunker.chunk(contentToProcess, `${result.url}.md`);
|
|
112
|
+
const fileType = classifyWebContentType(result.url, result.title);
|
|
113
|
+
const urlHash = createHash("md5").update(result.url).digest("hex");
|
|
114
|
+
for (const chunk of chunks) {
|
|
115
|
+
const chunkId = chunks.length > 1 ? `${store.id}-${urlHash}-${String(chunk.chunkIndex)}` : `${store.id}-${urlHash}`;
|
|
116
|
+
const vector = await services.embeddings.embed(chunk.content);
|
|
117
|
+
docs.push({
|
|
118
|
+
id: createDocumentId(chunkId),
|
|
119
|
+
content: chunk.content,
|
|
120
|
+
vector,
|
|
121
|
+
metadata: {
|
|
122
|
+
type: chunks.length > 1 ? "chunk" : "web",
|
|
123
|
+
storeId: store.id,
|
|
124
|
+
url: result.url,
|
|
125
|
+
title: result.title,
|
|
126
|
+
extracted: result.extracted !== void 0,
|
|
127
|
+
depth: result.depth,
|
|
128
|
+
indexedAt: /* @__PURE__ */ new Date(),
|
|
129
|
+
fileType,
|
|
130
|
+
chunkIndex: chunk.chunkIndex,
|
|
131
|
+
totalChunks: chunk.totalChunks,
|
|
132
|
+
sectionHeader: chunk.sectionHeader
|
|
133
|
+
}
|
|
134
|
+
});
|
|
135
|
+
chunksCreated++;
|
|
136
|
+
}
|
|
137
|
+
pagesIndexed++;
|
|
174
138
|
}
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
console.log(`Deleted store: ${s.name}`);
|
|
179
|
-
} else {
|
|
180
|
-
console.error(`Error: ${result.error.message}`);
|
|
181
|
-
process.exit(1);
|
|
182
|
-
}
|
|
183
|
-
} finally {
|
|
184
|
-
await destroyServices(services);
|
|
185
|
-
}
|
|
186
|
-
});
|
|
187
|
-
return store;
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
// src/cli/commands/search.ts
|
|
191
|
-
import { Command as Command3 } from "commander";
|
|
192
|
-
function createSearchCommand(getOptions) {
|
|
193
|
-
const search = new Command3("search").description("Search indexed documents using vector similarity + full-text matching").argument("<query>", "Search query").option("-s, --stores <stores>", "Limit search to specific stores (comma-separated IDs or names)").option("-m, --mode <mode>", "vector (embeddings only), fts (text only), hybrid (both, default)", "hybrid").option("-n, --limit <count>", "Maximum results to return (default: 10)", "10").option("-t, --threshold <score>", "Minimum score 0-1; omit low-relevance results").option("--include-content", "Show full document content, not just preview snippet").option("--detail <level>", "Context detail: minimal, contextual, full (default: minimal)", "minimal").action(async (query, options) => {
|
|
194
|
-
const globalOpts = getOptions();
|
|
195
|
-
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
196
|
-
try {
|
|
197
|
-
let storeIds = (await services.store.list()).map((s) => s.id);
|
|
198
|
-
if (options.stores !== void 0) {
|
|
199
|
-
const requestedStores = options.stores.split(",").map((s) => s.trim());
|
|
200
|
-
const resolvedStores = [];
|
|
201
|
-
for (const requested of requestedStores) {
|
|
202
|
-
const store = await services.store.getByIdOrName(requested);
|
|
203
|
-
if (store !== void 0) {
|
|
204
|
-
resolvedStores.push(store.id);
|
|
205
|
-
} else {
|
|
206
|
-
console.error(`Error: Store not found: ${requested}`);
|
|
207
|
-
process.exit(3);
|
|
139
|
+
if (docs.length > 0) {
|
|
140
|
+
if (spinner) {
|
|
141
|
+
spinner.text = "Indexing documents...";
|
|
208
142
|
}
|
|
143
|
+
await services.lance.addDocuments(store.id, docs);
|
|
209
144
|
}
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
for (const r of results.results) {
|
|
232
|
-
const path = r.metadata.path ?? r.metadata.url ?? "unknown";
|
|
233
|
-
console.log(path);
|
|
145
|
+
const crawlResult = {
|
|
146
|
+
success: true,
|
|
147
|
+
store: store.name,
|
|
148
|
+
storeCreated,
|
|
149
|
+
url,
|
|
150
|
+
pagesCrawled: pagesIndexed,
|
|
151
|
+
chunksCreated,
|
|
152
|
+
mode: cmdOptions.simple === true ? "simple" : "intelligent",
|
|
153
|
+
hadCrawlInstruction: cmdOptions.crawl !== void 0,
|
|
154
|
+
hadExtractInstruction: cmdOptions.extract !== void 0
|
|
155
|
+
};
|
|
156
|
+
if (globalOpts.format === "json") {
|
|
157
|
+
console.log(JSON.stringify(crawlResult, null, 2));
|
|
158
|
+
} else if (spinner !== void 0) {
|
|
159
|
+
spinner.succeed(
|
|
160
|
+
`Crawled ${String(pagesIndexed)} pages, indexed ${String(chunksCreated)} chunks`
|
|
161
|
+
);
|
|
162
|
+
} else if (globalOpts.quiet !== true) {
|
|
163
|
+
console.log(
|
|
164
|
+
`Crawled ${String(pagesIndexed)} pages, indexed ${String(chunksCreated)} chunks`
|
|
165
|
+
);
|
|
234
166
|
}
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
`);
|
|
240
|
-
if (results.results.length === 0) {
|
|
241
|
-
console.log("No results found.\n");
|
|
167
|
+
} catch (error) {
|
|
168
|
+
const message = `Crawl failed: ${error instanceof Error ? error.message : String(error)}`;
|
|
169
|
+
if (spinner) {
|
|
170
|
+
spinner.fail(message);
|
|
242
171
|
} else {
|
|
243
|
-
|
|
244
|
-
const r = results.results[i];
|
|
245
|
-
if (r === void 0) continue;
|
|
246
|
-
if (r.summary) {
|
|
247
|
-
console.log(`${String(i + 1)}. [${r.score.toFixed(2)}] ${r.summary.type}: ${r.summary.name}`);
|
|
248
|
-
console.log(` ${r.summary.location}`);
|
|
249
|
-
console.log(` ${r.summary.purpose}`);
|
|
250
|
-
if (r.context && options.detail !== "minimal") {
|
|
251
|
-
console.log(` Imports: ${r.context.keyImports.slice(0, 3).join(", ")}`);
|
|
252
|
-
console.log(` Related: ${r.context.relatedConcepts.slice(0, 3).join(", ")}`);
|
|
253
|
-
}
|
|
254
|
-
console.log();
|
|
255
|
-
} else {
|
|
256
|
-
const path = r.metadata.path ?? r.metadata.url ?? "unknown";
|
|
257
|
-
console.log(`${String(i + 1)}. [${r.score.toFixed(2)}] ${path}`);
|
|
258
|
-
const preview = r.highlight ?? r.content.slice(0, 150).replace(/\n/g, " ") + (r.content.length > 150 ? "..." : "");
|
|
259
|
-
console.log(` ${preview}
|
|
260
|
-
`);
|
|
261
|
-
}
|
|
262
|
-
}
|
|
172
|
+
console.error(`Error: ${message}`);
|
|
263
173
|
}
|
|
174
|
+
exitCode = 6;
|
|
175
|
+
} finally {
|
|
176
|
+
await crawler.stop();
|
|
177
|
+
await destroyServices(services);
|
|
178
|
+
}
|
|
179
|
+
if (exitCode !== 0) {
|
|
180
|
+
process.exit(exitCode);
|
|
264
181
|
}
|
|
265
|
-
} finally {
|
|
266
|
-
await destroyServices(services);
|
|
267
182
|
}
|
|
268
|
-
|
|
269
|
-
return search;
|
|
183
|
+
);
|
|
270
184
|
}
|
|
271
185
|
|
|
272
186
|
// src/cli/commands/index-cmd.ts
|
|
273
|
-
import { Command as
|
|
274
|
-
import
|
|
187
|
+
import { Command as Command2 } from "commander";
|
|
188
|
+
import ora2 from "ora";
|
|
275
189
|
function createIndexCommand(getOptions) {
|
|
276
|
-
const index = new
|
|
190
|
+
const index = new Command2("index").description("Scan store files, chunk text, generate embeddings, save to LanceDB").argument("<store>", "Store ID or name").option("--force", "Re-index all files even if unchanged").action(async (storeIdOrName, _options) => {
|
|
277
191
|
const globalOpts = getOptions();
|
|
278
192
|
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
193
|
+
let exitCode = 0;
|
|
279
194
|
try {
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
let spinner;
|
|
287
|
-
if (isInteractive) {
|
|
288
|
-
spinner = ora(`Indexing store: ${store.name}`).start();
|
|
289
|
-
} else if (globalOpts.quiet !== true && globalOpts.format !== "json") {
|
|
290
|
-
console.log(`Indexing store: ${store.name}`);
|
|
291
|
-
}
|
|
292
|
-
await services.lance.initialize(store.id);
|
|
293
|
-
const result = await services.index.indexStore(store, (event) => {
|
|
294
|
-
if (event.type === "progress") {
|
|
295
|
-
if (spinner) {
|
|
296
|
-
spinner.text = `Indexing: ${String(event.current)}/${String(event.total)} files - ${event.message}`;
|
|
297
|
-
}
|
|
195
|
+
indexLogic: {
|
|
196
|
+
const store = await services.store.getByIdOrName(storeIdOrName);
|
|
197
|
+
if (store === void 0) {
|
|
198
|
+
console.error(`Error: Store not found: ${storeIdOrName}`);
|
|
199
|
+
exitCode = 3;
|
|
200
|
+
break indexLogic;
|
|
298
201
|
}
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
if (
|
|
302
|
-
|
|
202
|
+
const isInteractive = process.stdout.isTTY && globalOpts.quiet !== true && globalOpts.format !== "json";
|
|
203
|
+
let spinner;
|
|
204
|
+
if (isInteractive) {
|
|
205
|
+
spinner = ora2(`Indexing store: ${store.name}`).start();
|
|
206
|
+
} else if (globalOpts.quiet !== true && globalOpts.format !== "json") {
|
|
207
|
+
console.log(`Indexing store: ${store.name}`);
|
|
208
|
+
}
|
|
209
|
+
await services.lance.initialize(store.id);
|
|
210
|
+
const result = await services.index.indexStore(store, (event) => {
|
|
211
|
+
if (event.type === "progress") {
|
|
212
|
+
if (spinner) {
|
|
213
|
+
spinner.text = `Indexing: ${String(event.current)}/${String(event.total)} files - ${event.message}`;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
});
|
|
217
|
+
if (result.success) {
|
|
218
|
+
if (globalOpts.format === "json") {
|
|
219
|
+
console.log(JSON.stringify(result.data, null, 2));
|
|
220
|
+
} else {
|
|
221
|
+
const message = `Indexed ${String(result.data.documentsIndexed)} documents, ${String(result.data.chunksCreated)} chunks in ${String(result.data.timeMs)}ms`;
|
|
222
|
+
if (spinner !== void 0) {
|
|
223
|
+
spinner.succeed(message);
|
|
224
|
+
} else if (globalOpts.quiet !== true) {
|
|
225
|
+
console.log(message);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
303
228
|
} else {
|
|
304
|
-
const message = `
|
|
229
|
+
const message = `Error: ${result.error.message}`;
|
|
305
230
|
if (spinner !== void 0) {
|
|
306
|
-
spinner.
|
|
307
|
-
} else
|
|
308
|
-
console.
|
|
231
|
+
spinner.fail(message);
|
|
232
|
+
} else {
|
|
233
|
+
console.error(message);
|
|
309
234
|
}
|
|
235
|
+
exitCode = 4;
|
|
236
|
+
break indexLogic;
|
|
310
237
|
}
|
|
311
|
-
} else {
|
|
312
|
-
const message = `Error: ${result.error.message}`;
|
|
313
|
-
if (spinner !== void 0) {
|
|
314
|
-
spinner.fail(message);
|
|
315
|
-
} else {
|
|
316
|
-
console.error(message);
|
|
317
|
-
}
|
|
318
|
-
process.exit(4);
|
|
319
238
|
}
|
|
320
239
|
} finally {
|
|
321
240
|
await destroyServices(services);
|
|
322
241
|
}
|
|
242
|
+
if (exitCode !== 0) {
|
|
243
|
+
process.exit(exitCode);
|
|
244
|
+
}
|
|
323
245
|
});
|
|
324
|
-
index.command("watch <store>").description("Watch store directory; re-index when files change").option(
|
|
246
|
+
index.command("watch <store>").description("Watch store directory; re-index when files change").option(
|
|
247
|
+
"--debounce <ms>",
|
|
248
|
+
"Wait N ms after last change before re-indexing (default: 1000)",
|
|
249
|
+
"1000"
|
|
250
|
+
).action(async (storeIdOrName, options) => {
|
|
325
251
|
const globalOpts = getOptions();
|
|
326
252
|
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
327
253
|
const store = await services.store.getByIdOrName(storeIdOrName);
|
|
@@ -329,7 +255,7 @@ function createIndexCommand(getOptions) {
|
|
|
329
255
|
console.error(`Error: File/repo store not found: ${storeIdOrName}`);
|
|
330
256
|
process.exit(3);
|
|
331
257
|
}
|
|
332
|
-
const { WatchService } = await import("./watch.service-
|
|
258
|
+
const { WatchService } = await import("./watch.service-BJV3TI3F.js");
|
|
333
259
|
const watchService = new WatchService(services.index, services.lance);
|
|
334
260
|
if (globalOpts.quiet !== true) {
|
|
335
261
|
console.log(`Watching ${store.name} for changes...`);
|
|
@@ -350,452 +276,29 @@ function createIndexCommand(getOptions) {
|
|
|
350
276
|
return index;
|
|
351
277
|
}
|
|
352
278
|
|
|
353
|
-
// src/cli/commands/
|
|
354
|
-
import { Command as
|
|
355
|
-
|
|
279
|
+
// src/cli/commands/mcp.ts
|
|
280
|
+
import { Command as Command3 } from "commander";
|
|
281
|
+
function createMCPCommand(getOptions) {
|
|
282
|
+
const mcp = new Command3("mcp").description("Start MCP (Model Context Protocol) server for AI agent integration").action(async () => {
|
|
283
|
+
const opts = getOptions();
|
|
284
|
+
await runMCPServer({
|
|
285
|
+
dataDir: opts.dataDir,
|
|
286
|
+
config: opts.config
|
|
287
|
+
});
|
|
288
|
+
});
|
|
289
|
+
return mcp;
|
|
290
|
+
}
|
|
356
291
|
|
|
357
|
-
// src/
|
|
358
|
-
import {
|
|
359
|
-
import { cors } from "hono/cors";
|
|
360
|
-
import { z } from "zod";
|
|
361
|
-
var CreateStoreBodySchema = z.object({
|
|
362
|
-
name: z.string().min(1, "Store name must be a non-empty string"),
|
|
363
|
-
type: z.enum(["file", "repo", "web"]),
|
|
364
|
-
path: z.string().min(1).optional(),
|
|
365
|
-
url: z.string().min(1).optional(),
|
|
366
|
-
description: z.string().optional(),
|
|
367
|
-
tags: z.array(z.string()).optional(),
|
|
368
|
-
branch: z.string().optional(),
|
|
369
|
-
depth: z.number().int().positive().optional()
|
|
370
|
-
}).refine(
|
|
371
|
-
(data) => {
|
|
372
|
-
switch (data.type) {
|
|
373
|
-
case "file":
|
|
374
|
-
return data.path !== void 0;
|
|
375
|
-
case "web":
|
|
376
|
-
return data.url !== void 0;
|
|
377
|
-
case "repo":
|
|
378
|
-
return data.path !== void 0 || data.url !== void 0;
|
|
379
|
-
}
|
|
380
|
-
},
|
|
381
|
-
{ message: "Missing required field: file stores need path, web stores need url, repo stores need path or url" }
|
|
382
|
-
);
|
|
383
|
-
var SearchBodySchema = z.object({
|
|
384
|
-
query: z.string().min(1, "Query must be a non-empty string"),
|
|
385
|
-
detail: z.enum(["minimal", "contextual", "full"]).optional(),
|
|
386
|
-
limit: z.number().int().positive().optional(),
|
|
387
|
-
stores: z.array(z.string()).optional()
|
|
388
|
-
});
|
|
389
|
-
function createApp(services) {
|
|
390
|
-
const app = new Hono();
|
|
391
|
-
app.use("*", cors());
|
|
392
|
-
app.get("/health", (c) => c.json({ status: "ok" }));
|
|
393
|
-
app.get("/api/stores", async (c) => {
|
|
394
|
-
const stores = await services.store.list();
|
|
395
|
-
return c.json(stores);
|
|
396
|
-
});
|
|
397
|
-
app.post("/api/stores", async (c) => {
|
|
398
|
-
const jsonData = await c.req.json();
|
|
399
|
-
const parseResult = CreateStoreBodySchema.safeParse(jsonData);
|
|
400
|
-
if (!parseResult.success) {
|
|
401
|
-
return c.json({ error: parseResult.error.issues[0]?.message ?? "Invalid request body" }, 400);
|
|
402
|
-
}
|
|
403
|
-
const result = await services.store.create(parseResult.data);
|
|
404
|
-
if (result.success) {
|
|
405
|
-
return c.json(result.data, 201);
|
|
406
|
-
}
|
|
407
|
-
return c.json({ error: result.error.message }, 400);
|
|
408
|
-
});
|
|
409
|
-
app.get("/api/stores/:id", async (c) => {
|
|
410
|
-
const store = await services.store.getByIdOrName(c.req.param("id"));
|
|
411
|
-
if (!store) return c.json({ error: "Not found" }, 404);
|
|
412
|
-
return c.json(store);
|
|
413
|
-
});
|
|
414
|
-
app.delete("/api/stores/:id", async (c) => {
|
|
415
|
-
const store = await services.store.getByIdOrName(c.req.param("id"));
|
|
416
|
-
if (!store) return c.json({ error: "Not found" }, 404);
|
|
417
|
-
const result = await services.store.delete(store.id);
|
|
418
|
-
if (result.success) return c.json({ deleted: true });
|
|
419
|
-
return c.json({ error: result.error.message }, 400);
|
|
420
|
-
});
|
|
421
|
-
app.post("/api/search", async (c) => {
|
|
422
|
-
const jsonData = await c.req.json();
|
|
423
|
-
const parseResult = SearchBodySchema.safeParse(jsonData);
|
|
424
|
-
if (!parseResult.success) {
|
|
425
|
-
return c.json({ error: parseResult.error.issues[0]?.message ?? "Invalid request body" }, 400);
|
|
426
|
-
}
|
|
427
|
-
const storeIds = (await services.store.list()).map((s) => s.id);
|
|
428
|
-
for (const id of storeIds) {
|
|
429
|
-
await services.lance.initialize(id);
|
|
430
|
-
}
|
|
431
|
-
const requestedStores = parseResult.data.stores !== void 0 ? parseResult.data.stores.map((s) => createStoreId(s)) : storeIds;
|
|
432
|
-
const query = {
|
|
433
|
-
query: parseResult.data.query,
|
|
434
|
-
detail: parseResult.data.detail ?? "minimal",
|
|
435
|
-
limit: parseResult.data.limit ?? 10,
|
|
436
|
-
stores: requestedStores
|
|
437
|
-
};
|
|
438
|
-
const results = await services.search.search(query);
|
|
439
|
-
return c.json(results);
|
|
440
|
-
});
|
|
441
|
-
app.post("/api/stores/:id/index", async (c) => {
|
|
442
|
-
const store = await services.store.getByIdOrName(c.req.param("id"));
|
|
443
|
-
if (!store) return c.json({ error: "Not found" }, 404);
|
|
444
|
-
await services.lance.initialize(store.id);
|
|
445
|
-
const result = await services.index.indexStore(store);
|
|
446
|
-
if (result.success) return c.json(result.data);
|
|
447
|
-
return c.json({ error: result.error.message }, 400);
|
|
448
|
-
});
|
|
449
|
-
return app;
|
|
450
|
-
}
|
|
451
|
-
|
|
452
|
-
// src/cli/commands/serve.ts
|
|
453
|
-
function createServeCommand(getOptions) {
|
|
454
|
-
return new Command5("serve").description("Start HTTP API server for programmatic search access").option("-p, --port <port>", "Port to listen on (default: 3847)", "3847").option("--host <host>", "Bind address (default: 127.0.0.1, use 0.0.0.0 for all interfaces)", "127.0.0.1").action(async (options) => {
|
|
455
|
-
const globalOpts = getOptions();
|
|
456
|
-
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
457
|
-
const app = createApp(services);
|
|
458
|
-
const port = parseInt(options.port ?? "3847", 10);
|
|
459
|
-
const host = options.host ?? "127.0.0.1";
|
|
460
|
-
console.log(`Starting server on http://${host}:${String(port)}`);
|
|
461
|
-
serve({
|
|
462
|
-
fetch: app.fetch,
|
|
463
|
-
port,
|
|
464
|
-
hostname: host
|
|
465
|
-
});
|
|
466
|
-
});
|
|
467
|
-
}
|
|
468
|
-
|
|
469
|
-
// src/cli/commands/crawl.ts
|
|
470
|
-
import { Command as Command6 } from "commander";
|
|
471
|
-
import { createHash } from "crypto";
|
|
472
|
-
import ora2 from "ora";
|
|
473
|
-
function createCrawlCommand(getOptions) {
|
|
474
|
-
return new Command6("crawl").description("Crawl web pages with natural language control and index into store").argument("<url>", "URL to crawl").argument("<store>", "Target web store to add crawled content to").option("--crawl <instruction>", 'Natural language instruction for what to crawl (e.g., "all Getting Started pages")').option("--extract <instruction>", 'Natural language instruction for what to extract (e.g., "extract API references")').option("--simple", "Use simple BFS mode instead of intelligent crawling").option("--max-pages <number>", "Maximum number of pages to crawl", "50").option("--headless", "Use headless browser for JavaScript-rendered sites").action(async (url, storeIdOrName, cmdOptions) => {
|
|
475
|
-
const globalOpts = getOptions();
|
|
476
|
-
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
477
|
-
let store;
|
|
478
|
-
let storeCreated = false;
|
|
479
|
-
const existingStore = await services.store.getByIdOrName(storeIdOrName);
|
|
480
|
-
if (!existingStore) {
|
|
481
|
-
const result = await services.store.create({
|
|
482
|
-
name: storeIdOrName,
|
|
483
|
-
type: "web",
|
|
484
|
-
url
|
|
485
|
-
});
|
|
486
|
-
if (!result.success) {
|
|
487
|
-
await destroyServices(services);
|
|
488
|
-
throw new Error(`Failed to create store: ${result.error.message}`);
|
|
489
|
-
}
|
|
490
|
-
const createdStore = result.data;
|
|
491
|
-
if (createdStore.type !== "web") {
|
|
492
|
-
throw new Error("Unexpected store type after creation");
|
|
493
|
-
}
|
|
494
|
-
store = createdStore;
|
|
495
|
-
storeCreated = true;
|
|
496
|
-
if (globalOpts.quiet !== true && globalOpts.format !== "json") {
|
|
497
|
-
console.log(`Created web store: ${store.name}`);
|
|
498
|
-
}
|
|
499
|
-
} else if (existingStore.type !== "web") {
|
|
500
|
-
await destroyServices(services);
|
|
501
|
-
throw new Error(`Store "${storeIdOrName}" exists but is not a web store (type: ${existingStore.type})`);
|
|
502
|
-
} else {
|
|
503
|
-
store = existingStore;
|
|
504
|
-
}
|
|
505
|
-
const maxPages = cmdOptions.maxPages !== void 0 ? parseInt(cmdOptions.maxPages) : 50;
|
|
506
|
-
const isInteractive = process.stdout.isTTY && globalOpts.quiet !== true && globalOpts.format !== "json";
|
|
507
|
-
let spinner;
|
|
508
|
-
if (isInteractive) {
|
|
509
|
-
const mode = cmdOptions.simple === true ? "simple" : "intelligent";
|
|
510
|
-
spinner = ora2(`Crawling ${url} (${mode} mode)`).start();
|
|
511
|
-
} else if (globalOpts.quiet !== true && globalOpts.format !== "json") {
|
|
512
|
-
console.log(`Crawling ${url}`);
|
|
513
|
-
}
|
|
514
|
-
const crawler = new IntelligentCrawler();
|
|
515
|
-
const webChunker = ChunkingService.forContentType("web");
|
|
516
|
-
let pagesIndexed = 0;
|
|
517
|
-
let chunksCreated = 0;
|
|
518
|
-
crawler.on("progress", (progress) => {
|
|
519
|
-
if (spinner) {
|
|
520
|
-
if (progress.type === "strategy") {
|
|
521
|
-
spinner.text = progress.message !== void 0 ? progress.message : "Analyzing crawl strategy...";
|
|
522
|
-
} else if (progress.type === "page") {
|
|
523
|
-
const url2 = progress.currentUrl !== void 0 ? progress.currentUrl : "unknown";
|
|
524
|
-
spinner.text = `Crawling ${String(progress.pagesVisited + 1)}/${String(maxPages)} - ${url2}`;
|
|
525
|
-
} else if (progress.type === "extraction") {
|
|
526
|
-
const url2 = progress.currentUrl !== void 0 ? progress.currentUrl : "unknown";
|
|
527
|
-
spinner.text = `Extracting from ${url2}...`;
|
|
528
|
-
} else if (progress.type === "error" && progress.message !== void 0) {
|
|
529
|
-
spinner.warn(progress.message);
|
|
530
|
-
}
|
|
531
|
-
}
|
|
532
|
-
});
|
|
533
|
-
try {
|
|
534
|
-
await services.lance.initialize(store.id);
|
|
535
|
-
const docs = [];
|
|
536
|
-
for await (const result of crawler.crawl(url, {
|
|
537
|
-
...cmdOptions.crawl !== void 0 && { crawlInstruction: cmdOptions.crawl },
|
|
538
|
-
...cmdOptions.extract !== void 0 && { extractInstruction: cmdOptions.extract },
|
|
539
|
-
maxPages,
|
|
540
|
-
...cmdOptions.simple !== void 0 && { simple: cmdOptions.simple },
|
|
541
|
-
useHeadless: cmdOptions.headless ?? false
|
|
542
|
-
})) {
|
|
543
|
-
const contentToProcess = result.extracted !== void 0 ? result.extracted : result.markdown;
|
|
544
|
-
const chunks = webChunker.chunk(contentToProcess, `${result.url}.md`);
|
|
545
|
-
const fileType = classifyWebContentType(result.url, result.title);
|
|
546
|
-
const urlHash = createHash("md5").update(result.url).digest("hex");
|
|
547
|
-
for (const chunk of chunks) {
|
|
548
|
-
const chunkId = chunks.length > 1 ? `${store.id}-${urlHash}-${String(chunk.chunkIndex)}` : `${store.id}-${urlHash}`;
|
|
549
|
-
const vector = await services.embeddings.embed(chunk.content);
|
|
550
|
-
docs.push({
|
|
551
|
-
id: createDocumentId(chunkId),
|
|
552
|
-
content: chunk.content,
|
|
553
|
-
vector,
|
|
554
|
-
metadata: {
|
|
555
|
-
type: chunks.length > 1 ? "chunk" : "web",
|
|
556
|
-
storeId: store.id,
|
|
557
|
-
url: result.url,
|
|
558
|
-
title: result.title,
|
|
559
|
-
extracted: result.extracted !== void 0,
|
|
560
|
-
depth: result.depth,
|
|
561
|
-
indexedAt: /* @__PURE__ */ new Date(),
|
|
562
|
-
fileType,
|
|
563
|
-
chunkIndex: chunk.chunkIndex,
|
|
564
|
-
totalChunks: chunk.totalChunks,
|
|
565
|
-
sectionHeader: chunk.sectionHeader
|
|
566
|
-
}
|
|
567
|
-
});
|
|
568
|
-
chunksCreated++;
|
|
569
|
-
}
|
|
570
|
-
pagesIndexed++;
|
|
571
|
-
}
|
|
572
|
-
if (docs.length > 0) {
|
|
573
|
-
if (spinner) {
|
|
574
|
-
spinner.text = "Indexing documents...";
|
|
575
|
-
}
|
|
576
|
-
await services.lance.addDocuments(store.id, docs);
|
|
577
|
-
}
|
|
578
|
-
const crawlResult = {
|
|
579
|
-
success: true,
|
|
580
|
-
store: store.name,
|
|
581
|
-
storeCreated,
|
|
582
|
-
url,
|
|
583
|
-
pagesCrawled: pagesIndexed,
|
|
584
|
-
chunksCreated,
|
|
585
|
-
mode: cmdOptions.simple === true ? "simple" : "intelligent",
|
|
586
|
-
hadCrawlInstruction: cmdOptions.crawl !== void 0,
|
|
587
|
-
hadExtractInstruction: cmdOptions.extract !== void 0
|
|
588
|
-
};
|
|
589
|
-
if (globalOpts.format === "json") {
|
|
590
|
-
console.log(JSON.stringify(crawlResult, null, 2));
|
|
591
|
-
} else if (spinner !== void 0) {
|
|
592
|
-
spinner.succeed(`Crawled ${String(pagesIndexed)} pages, indexed ${String(chunksCreated)} chunks`);
|
|
593
|
-
} else if (globalOpts.quiet !== true) {
|
|
594
|
-
console.log(`Crawled ${String(pagesIndexed)} pages, indexed ${String(chunksCreated)} chunks`);
|
|
595
|
-
}
|
|
596
|
-
} catch (error) {
|
|
597
|
-
const message = `Crawl failed: ${error instanceof Error ? error.message : String(error)}`;
|
|
598
|
-
if (spinner) {
|
|
599
|
-
spinner.fail(message);
|
|
600
|
-
} else {
|
|
601
|
-
console.error(`Error: ${message}`);
|
|
602
|
-
}
|
|
603
|
-
process.exit(6);
|
|
604
|
-
} finally {
|
|
605
|
-
await crawler.stop();
|
|
606
|
-
await destroyServices(services);
|
|
607
|
-
}
|
|
608
|
-
});
|
|
609
|
-
}
|
|
292
|
+
// src/cli/commands/plugin-api.ts
|
|
293
|
+
import { Command as Command4 } from "commander";
|
|
610
294
|
|
|
611
|
-
// src/
|
|
612
|
-
import { Command as Command7 } from "commander";
|
|
613
|
-
import { spawnSync } from "child_process";
|
|
614
|
-
import { existsSync } from "fs";
|
|
615
|
-
import { mkdir } from "fs/promises";
|
|
616
|
-
import { join as join2 } from "path";
|
|
617
|
-
import { homedir } from "os";
|
|
295
|
+
// src/plugin/commands.ts
|
|
618
296
|
import ora3 from "ora";
|
|
619
297
|
|
|
620
|
-
// src/defaults/repos.ts
|
|
621
|
-
var DEFAULT_REPOS = [
|
|
622
|
-
{
|
|
623
|
-
url: "git@github.com:ericbuess/claude-code-docs.git",
|
|
624
|
-
name: "claude-code-docs",
|
|
625
|
-
description: "Claude Code documentation",
|
|
626
|
-
tags: ["claude", "docs", "claude-code"]
|
|
627
|
-
},
|
|
628
|
-
{
|
|
629
|
-
url: "git@github.com:anthropics/claude-code.git",
|
|
630
|
-
name: "claude-code",
|
|
631
|
-
description: "Claude Code CLI tool source",
|
|
632
|
-
tags: ["claude", "cli", "anthropic"]
|
|
633
|
-
},
|
|
634
|
-
{
|
|
635
|
-
url: "git@github.com:anthropics/claude-agent-sdk-python.git",
|
|
636
|
-
name: "claude-agent-sdk-python",
|
|
637
|
-
description: "Claude Agent SDK for Python",
|
|
638
|
-
tags: ["claude", "sdk", "python", "agents"]
|
|
639
|
-
},
|
|
640
|
-
{
|
|
641
|
-
url: "git@github.com:anthropics/skills.git",
|
|
642
|
-
name: "claude-skills",
|
|
643
|
-
description: "Claude skills and capabilities",
|
|
644
|
-
tags: ["claude", "skills"]
|
|
645
|
-
},
|
|
646
|
-
{
|
|
647
|
-
url: "git@github.com:anthropics/claude-quickstarts.git",
|
|
648
|
-
name: "claude-quickstarts",
|
|
649
|
-
description: "Claude quickstart examples and tutorials",
|
|
650
|
-
tags: ["claude", "examples", "tutorials"]
|
|
651
|
-
},
|
|
652
|
-
{
|
|
653
|
-
url: "git@github.com:anthropics/claude-plugins-official.git",
|
|
654
|
-
name: "claude-plugins",
|
|
655
|
-
description: "Official Claude plugins",
|
|
656
|
-
tags: ["claude", "plugins"]
|
|
657
|
-
},
|
|
658
|
-
{
|
|
659
|
-
url: "git@github.com:anthropics/claude-agent-sdk-typescript.git",
|
|
660
|
-
name: "claude-agent-sdk-typescript",
|
|
661
|
-
description: "Claude Agent SDK for TypeScript",
|
|
662
|
-
tags: ["claude", "sdk", "typescript", "agents"]
|
|
663
|
-
},
|
|
664
|
-
{
|
|
665
|
-
url: "git@github.com:anthropics/claude-agent-sdk-demos.git",
|
|
666
|
-
name: "claude-agent-sdk-demos",
|
|
667
|
-
description: "Claude Agent SDK demo applications",
|
|
668
|
-
tags: ["claude", "sdk", "demos", "examples"]
|
|
669
|
-
}
|
|
670
|
-
];
|
|
671
|
-
|
|
672
|
-
// src/cli/commands/setup.ts
|
|
673
|
-
var DEFAULT_REPOS_DIR = join2(homedir(), ".bluera", "bluera-knowledge", "repos");
|
|
674
|
-
function createSetupCommand(getOptions) {
|
|
675
|
-
const setup = new Command7("setup").description("Quick-start with pre-configured Claude/Anthropic documentation repos");
|
|
676
|
-
setup.command("repos").description("Clone repos to ~/.bluera/bluera-knowledge/repos/, create stores, index all content").option("--repos-dir <path>", "Clone destination (default: ~/.bluera/bluera-knowledge/repos/)", DEFAULT_REPOS_DIR).option("--skip-clone", "Don't clone; assume repos already exist locally").option("--skip-index", "Clone and create stores but don't index yet").option("--only <names>", "Only process matching repos (comma-separated, partial match)").option("--list", "Print available repos without cloning/indexing").action(async (options) => {
|
|
677
|
-
const globalOpts = getOptions();
|
|
678
|
-
if (options.list === true) {
|
|
679
|
-
console.log("\nDefault repositories:\n");
|
|
680
|
-
for (const repo of DEFAULT_REPOS) {
|
|
681
|
-
console.log(` ${repo.name}`);
|
|
682
|
-
console.log(` URL: ${repo.url}`);
|
|
683
|
-
console.log(` Description: ${repo.description}`);
|
|
684
|
-
console.log(` Tags: ${repo.tags.join(", ")}`);
|
|
685
|
-
console.log("");
|
|
686
|
-
}
|
|
687
|
-
return;
|
|
688
|
-
}
|
|
689
|
-
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
690
|
-
try {
|
|
691
|
-
let repos = DEFAULT_REPOS;
|
|
692
|
-
if (options.only !== void 0 && options.only !== "") {
|
|
693
|
-
const onlyNames = options.only.split(",").map((n) => n.trim().toLowerCase());
|
|
694
|
-
repos = DEFAULT_REPOS.filter(
|
|
695
|
-
(r) => onlyNames.some((n) => r.name.toLowerCase().includes(n))
|
|
696
|
-
);
|
|
697
|
-
if (repos.length === 0) {
|
|
698
|
-
console.error(`No repos matched: ${options.only}`);
|
|
699
|
-
console.log("Available repos:", DEFAULT_REPOS.map((r) => r.name).join(", "));
|
|
700
|
-
process.exit(1);
|
|
701
|
-
}
|
|
702
|
-
}
|
|
703
|
-
console.log(`
|
|
704
|
-
Setting up ${String(repos.length)} repositories...
|
|
705
|
-
`);
|
|
706
|
-
await mkdir(options.reposDir, { recursive: true });
|
|
707
|
-
for (const repo of repos) {
|
|
708
|
-
const repoPath = join2(options.reposDir, repo.name);
|
|
709
|
-
const spinner = ora3(`Processing ${repo.name}`).start();
|
|
710
|
-
try {
|
|
711
|
-
if (options.skipClone !== true) {
|
|
712
|
-
if (existsSync(repoPath)) {
|
|
713
|
-
spinner.text = `${repo.name}: Already cloned, pulling latest...`;
|
|
714
|
-
const pullResult = spawnSync("git", ["pull", "--ff-only"], { cwd: repoPath, stdio: "pipe" });
|
|
715
|
-
if (pullResult.status !== 0) {
|
|
716
|
-
spinner.text = `${repo.name}: Pull skipped (local changes)`;
|
|
717
|
-
}
|
|
718
|
-
} else {
|
|
719
|
-
spinner.text = `${repo.name}: Cloning...`;
|
|
720
|
-
const cloneResult = spawnSync("git", ["clone", repo.url, repoPath], { stdio: "pipe" });
|
|
721
|
-
if (cloneResult.status !== 0) {
|
|
722
|
-
const errorMessage = cloneResult.stderr.length > 0 ? cloneResult.stderr.toString() : "Git clone failed";
|
|
723
|
-
throw new Error(errorMessage);
|
|
724
|
-
}
|
|
725
|
-
}
|
|
726
|
-
}
|
|
727
|
-
spinner.text = `${repo.name}: Creating store...`;
|
|
728
|
-
const existingStore = await services.store.getByIdOrName(repo.name);
|
|
729
|
-
let storeId;
|
|
730
|
-
if (existingStore) {
|
|
731
|
-
storeId = existingStore.id;
|
|
732
|
-
spinner.text = `${repo.name}: Store already exists`;
|
|
733
|
-
} else {
|
|
734
|
-
const result = await services.store.create({
|
|
735
|
-
name: repo.name,
|
|
736
|
-
type: "repo",
|
|
737
|
-
path: repoPath,
|
|
738
|
-
description: repo.description,
|
|
739
|
-
tags: repo.tags
|
|
740
|
-
});
|
|
741
|
-
if (!result.success) {
|
|
742
|
-
throw new Error(result.error instanceof Error ? result.error.message : String(result.error));
|
|
743
|
-
}
|
|
744
|
-
storeId = result.data.id;
|
|
745
|
-
}
|
|
746
|
-
if (options.skipIndex !== true) {
|
|
747
|
-
spinner.text = `${repo.name}: Indexing...`;
|
|
748
|
-
const store = await services.store.getByIdOrName(storeId);
|
|
749
|
-
if (store) {
|
|
750
|
-
await services.lance.initialize(store.id);
|
|
751
|
-
const indexResult = await services.index.indexStore(store, (event) => {
|
|
752
|
-
if (event.type === "progress") {
|
|
753
|
-
spinner.text = `${repo.name}: Indexing ${String(event.current)}/${String(event.total)} files`;
|
|
754
|
-
}
|
|
755
|
-
});
|
|
756
|
-
if (indexResult.success) {
|
|
757
|
-
spinner.succeed(
|
|
758
|
-
`${repo.name}: ${String(indexResult.data.documentsIndexed)} docs, ${String(indexResult.data.chunksCreated)} chunks`
|
|
759
|
-
);
|
|
760
|
-
} else {
|
|
761
|
-
throw new Error(indexResult.error instanceof Error ? indexResult.error.message : String(indexResult.error));
|
|
762
|
-
}
|
|
763
|
-
}
|
|
764
|
-
} else {
|
|
765
|
-
spinner.succeed(`${repo.name}: Ready (indexing skipped)`);
|
|
766
|
-
}
|
|
767
|
-
} catch (error) {
|
|
768
|
-
spinner.fail(`${repo.name}: ${error instanceof Error ? error.message : String(error)}`);
|
|
769
|
-
}
|
|
770
|
-
}
|
|
771
|
-
console.log('\nSetup complete! Use "bluera-knowledge search <query>" to search.\n');
|
|
772
|
-
} finally {
|
|
773
|
-
await destroyServices(services);
|
|
774
|
-
}
|
|
775
|
-
});
|
|
776
|
-
return setup;
|
|
777
|
-
}
|
|
778
|
-
|
|
779
|
-
// src/cli/commands/mcp.ts
|
|
780
|
-
import { Command as Command8 } from "commander";
|
|
781
|
-
function createMCPCommand(getOptions) {
|
|
782
|
-
const mcp = new Command8("mcp").description("Start MCP (Model Context Protocol) server for AI agent integration").action(async () => {
|
|
783
|
-
const opts = getOptions();
|
|
784
|
-
await runMCPServer({
|
|
785
|
-
dataDir: opts.dataDir,
|
|
786
|
-
config: opts.config
|
|
787
|
-
});
|
|
788
|
-
});
|
|
789
|
-
return mcp;
|
|
790
|
-
}
|
|
791
|
-
|
|
792
|
-
// src/cli/commands/plugin-api.ts
|
|
793
|
-
import { Command as Command9 } from "commander";
|
|
794
|
-
|
|
795
298
|
// src/analysis/dependency-usage-analyzer.ts
|
|
299
|
+
import { existsSync } from "fs";
|
|
796
300
|
import { readFile, readdir } from "fs/promises";
|
|
797
|
-
import {
|
|
798
|
-
import { join as join3, extname } from "path";
|
|
301
|
+
import { join, extname } from "path";
|
|
799
302
|
var TEXT_EXTENSIONS = /* @__PURE__ */ new Set([
|
|
800
303
|
".ts",
|
|
801
304
|
".tsx",
|
|
@@ -857,25 +360,25 @@ var DependencyUsageAnalyzer = class {
|
|
|
857
360
|
if (packageName !== null && declaredDeps.has(packageName)) {
|
|
858
361
|
const dep = declaredDeps.get(packageName);
|
|
859
362
|
if (dep !== void 0) {
|
|
860
|
-
this.incrementUsage(
|
|
861
|
-
usageMap,
|
|
862
|
-
packageName,
|
|
863
|
-
filePath,
|
|
864
|
-
dep.isDev,
|
|
865
|
-
dep.language
|
|
866
|
-
);
|
|
363
|
+
this.incrementUsage(usageMap, packageName, filePath, dep.isDev, dep.language);
|
|
867
364
|
}
|
|
868
365
|
}
|
|
869
366
|
}
|
|
870
367
|
processedCount++;
|
|
871
368
|
if (onProgress !== void 0 && processedCount % 10 === 0) {
|
|
872
|
-
onProgress(
|
|
369
|
+
onProgress(
|
|
370
|
+
processedCount,
|
|
371
|
+
files.length,
|
|
372
|
+
`Analyzed ${String(processedCount)}/${String(files.length)} files`
|
|
373
|
+
);
|
|
873
374
|
}
|
|
874
375
|
} catch {
|
|
875
376
|
skippedCount++;
|
|
876
377
|
}
|
|
877
378
|
}
|
|
878
|
-
const sortedUsages = Array.from(usageMap.values()).sort(
|
|
379
|
+
const sortedUsages = Array.from(usageMap.values()).sort(
|
|
380
|
+
(a, b) => b.importCount - a.importCount
|
|
381
|
+
);
|
|
879
382
|
return ok({
|
|
880
383
|
usages: sortedUsages,
|
|
881
384
|
totalFilesScanned: processedCount,
|
|
@@ -968,9 +471,18 @@ var DependencyUsageAnalyzer = class {
|
|
|
968
471
|
try {
|
|
969
472
|
const entries = await readdir(dir, { withFileTypes: true });
|
|
970
473
|
for (const entry of entries) {
|
|
971
|
-
const fullPath =
|
|
474
|
+
const fullPath = join(dir, entry.name);
|
|
972
475
|
if (entry.isDirectory()) {
|
|
973
|
-
if (![
|
|
476
|
+
if (![
|
|
477
|
+
"node_modules",
|
|
478
|
+
".git",
|
|
479
|
+
"dist",
|
|
480
|
+
"build",
|
|
481
|
+
"coverage",
|
|
482
|
+
"__pycache__",
|
|
483
|
+
".venv",
|
|
484
|
+
"venv"
|
|
485
|
+
].includes(entry.name)) {
|
|
974
486
|
files.push(...await this.scanDirectory(fullPath));
|
|
975
487
|
}
|
|
976
488
|
} else if (entry.isFile()) {
|
|
@@ -986,8 +498,8 @@ var DependencyUsageAnalyzer = class {
|
|
|
986
498
|
}
|
|
987
499
|
async readDeclaredDependencies(projectRoot) {
|
|
988
500
|
const deps = /* @__PURE__ */ new Map();
|
|
989
|
-
const packageJsonPath =
|
|
990
|
-
if (
|
|
501
|
+
const packageJsonPath = join(projectRoot, "package.json");
|
|
502
|
+
if (existsSync(packageJsonPath)) {
|
|
991
503
|
try {
|
|
992
504
|
const content = await readFile(packageJsonPath, "utf-8");
|
|
993
505
|
const parsed = JSON.parse(content);
|
|
@@ -1006,8 +518,8 @@ var DependencyUsageAnalyzer = class {
|
|
|
1006
518
|
} catch {
|
|
1007
519
|
}
|
|
1008
520
|
}
|
|
1009
|
-
const reqPath =
|
|
1010
|
-
if (
|
|
521
|
+
const reqPath = join(projectRoot, "requirements.txt");
|
|
522
|
+
if (existsSync(reqPath)) {
|
|
1011
523
|
try {
|
|
1012
524
|
const content = await readFile(reqPath, "utf-8");
|
|
1013
525
|
const lines = content.split("\n");
|
|
@@ -1015,7 +527,7 @@ var DependencyUsageAnalyzer = class {
|
|
|
1015
527
|
const trimmed = line.trim();
|
|
1016
528
|
if (trimmed === "" || trimmed.startsWith("#")) continue;
|
|
1017
529
|
const match = /^([a-zA-Z0-9_-]+)/.exec(trimmed);
|
|
1018
|
-
if (match
|
|
530
|
+
if (match?.[1] !== void 0) {
|
|
1019
531
|
const name = match[1].toLowerCase();
|
|
1020
532
|
deps.set(name, { name, isDev: false, language: "python" });
|
|
1021
533
|
}
|
|
@@ -1023,8 +535,8 @@ var DependencyUsageAnalyzer = class {
|
|
|
1023
535
|
} catch {
|
|
1024
536
|
}
|
|
1025
537
|
}
|
|
1026
|
-
const pyprojectPath =
|
|
1027
|
-
if (
|
|
538
|
+
const pyprojectPath = join(projectRoot, "pyproject.toml");
|
|
539
|
+
if (existsSync(pyprojectPath)) {
|
|
1028
540
|
try {
|
|
1029
541
|
const content = await readFile(pyprojectPath, "utf-8");
|
|
1030
542
|
const depMatches = content.matchAll(/"([a-zA-Z0-9_-]+)"/g);
|
|
@@ -1037,13 +549,13 @@ var DependencyUsageAnalyzer = class {
|
|
|
1037
549
|
} catch {
|
|
1038
550
|
}
|
|
1039
551
|
}
|
|
1040
|
-
const cargoPath =
|
|
1041
|
-
if (
|
|
552
|
+
const cargoPath = join(projectRoot, "Cargo.toml");
|
|
553
|
+
if (existsSync(cargoPath)) {
|
|
1042
554
|
try {
|
|
1043
555
|
const content = await readFile(cargoPath, "utf-8");
|
|
1044
556
|
const inDepsSection = /\[dependencies\]([\s\S]*?)(?=\n\[|$)/;
|
|
1045
557
|
const depsMatch = inDepsSection.exec(content);
|
|
1046
|
-
if (depsMatch
|
|
558
|
+
if (depsMatch?.[1] !== void 0) {
|
|
1047
559
|
const depsSection = depsMatch[1];
|
|
1048
560
|
const cratePattern = /^([a-zA-Z0-9_-]+)\s*=/gm;
|
|
1049
561
|
for (const match of depsSection.matchAll(cratePattern)) {
|
|
@@ -1054,7 +566,7 @@ var DependencyUsageAnalyzer = class {
|
|
|
1054
566
|
}
|
|
1055
567
|
const inDevDepsSection = /\[dev-dependencies\]([\s\S]*?)(?=\n\[|$)/;
|
|
1056
568
|
const devDepsMatch = inDevDepsSection.exec(content);
|
|
1057
|
-
if (devDepsMatch
|
|
569
|
+
if (devDepsMatch?.[1] !== void 0) {
|
|
1058
570
|
const devDepsSection = devDepsMatch[1];
|
|
1059
571
|
const cratePattern = /^([a-zA-Z0-9_-]+)\s*=/gm;
|
|
1060
572
|
for (const match of devDepsSection.matchAll(cratePattern)) {
|
|
@@ -1066,8 +578,8 @@ var DependencyUsageAnalyzer = class {
|
|
|
1066
578
|
} catch {
|
|
1067
579
|
}
|
|
1068
580
|
}
|
|
1069
|
-
const goModPath =
|
|
1070
|
-
if (
|
|
581
|
+
const goModPath = join(projectRoot, "go.mod");
|
|
582
|
+
if (existsSync(goModPath)) {
|
|
1071
583
|
try {
|
|
1072
584
|
const content = await readFile(goModPath, "utf-8");
|
|
1073
585
|
const requirePattern = /^\s*([a-zA-Z0-9._/-]+)\s+v[\d.]+/gm;
|
|
@@ -1253,7 +765,6 @@ var RepoUrlResolver = class {
|
|
|
1253
765
|
};
|
|
1254
766
|
|
|
1255
767
|
// src/plugin/commands.ts
|
|
1256
|
-
import ora4 from "ora";
|
|
1257
768
|
async function handleAddRepo(args) {
|
|
1258
769
|
const services = await createServices(void 0, void 0, process.env["PWD"]);
|
|
1259
770
|
const storeName = args.name ?? extractRepoName(args.url);
|
|
@@ -1337,7 +848,7 @@ async function handleSuggest() {
|
|
|
1337
848
|
const services = await createServices(void 0, void 0, projectRoot);
|
|
1338
849
|
const analyzer = new DependencyUsageAnalyzer();
|
|
1339
850
|
const resolver = new RepoUrlResolver();
|
|
1340
|
-
const spinner =
|
|
851
|
+
const spinner = ora3("Scanning source files...").start();
|
|
1341
852
|
const result = await analyzer.analyze(projectRoot, (current, total, message) => {
|
|
1342
853
|
spinner.text = `${message} (${String(current)}/${String(total)})`;
|
|
1343
854
|
});
|
|
@@ -1347,8 +858,10 @@ async function handleSuggest() {
|
|
|
1347
858
|
process.exit(1);
|
|
1348
859
|
}
|
|
1349
860
|
const { usages, totalFilesScanned, skippedFiles } = result.data;
|
|
1350
|
-
console.log(
|
|
1351
|
-
`)
|
|
861
|
+
console.log(
|
|
862
|
+
`\u2714 Scanned ${String(totalFilesScanned)} files${skippedFiles > 0 ? ` (skipped ${String(skippedFiles)})` : ""}
|
|
863
|
+
`
|
|
864
|
+
);
|
|
1352
865
|
if (usages.length === 0) {
|
|
1353
866
|
console.log("No external dependencies found in this project.");
|
|
1354
867
|
console.log("\nMake sure you have a package.json or requirements.txt file.");
|
|
@@ -1365,23 +878,24 @@ async function handleSuggest() {
|
|
|
1365
878
|
console.log("Top dependencies by usage in this project:\n");
|
|
1366
879
|
topSuggestions.forEach((usage, i) => {
|
|
1367
880
|
console.log(`${String(i + 1)}. ${usage.packageName}`);
|
|
1368
|
-
console.log(
|
|
1369
|
-
`)
|
|
881
|
+
console.log(
|
|
882
|
+
` ${String(usage.importCount)} imports across ${String(usage.fileCount)} files
|
|
883
|
+
`
|
|
884
|
+
);
|
|
1370
885
|
});
|
|
1371
886
|
console.log("Searching for repository URLs...\n");
|
|
1372
887
|
for (const usage of topSuggestions) {
|
|
1373
|
-
const repoResult = await resolver.findRepoUrl(
|
|
1374
|
-
usage.packageName,
|
|
1375
|
-
usage.language
|
|
1376
|
-
);
|
|
888
|
+
const repoResult = await resolver.findRepoUrl(usage.packageName, usage.language);
|
|
1377
889
|
if (repoResult.url !== null) {
|
|
1378
890
|
console.log(`\u2714 ${usage.packageName}: ${repoResult.url}`);
|
|
1379
891
|
console.log(` /bluera-knowledge:add-repo ${repoResult.url} --name=${usage.packageName}
|
|
1380
892
|
`);
|
|
1381
893
|
} else {
|
|
1382
894
|
console.log(`\u2717 ${usage.packageName}: Could not find repository URL`);
|
|
1383
|
-
console.log(
|
|
1384
|
-
`
|
|
895
|
+
console.log(
|
|
896
|
+
` You can manually add it: /bluera-knowledge:add-repo <url> --name=${usage.packageName}
|
|
897
|
+
`
|
|
898
|
+
);
|
|
1385
899
|
}
|
|
1386
900
|
}
|
|
1387
901
|
console.log("Use the commands above to add repositories to your knowledge stores.");
|
|
@@ -1389,26 +903,627 @@ async function handleSuggest() {
|
|
|
1389
903
|
|
|
1390
904
|
// src/cli/commands/plugin-api.ts
|
|
1391
905
|
function createAddRepoCommand(_getOptions) {
|
|
1392
|
-
return new
|
|
906
|
+
return new Command4("add-repo").description("Clone and index a library source repository").argument("<url>", "Git repository URL").option("--name <name>", "Store name (defaults to repo name)").option("--branch <branch>", "Git branch to clone").action(async (url, options) => {
|
|
1393
907
|
await handleAddRepo({ url, ...options });
|
|
1394
908
|
});
|
|
1395
909
|
}
|
|
1396
910
|
function createAddFolderCommand(_getOptions) {
|
|
1397
|
-
return new
|
|
911
|
+
return new Command4("add-folder").description("Index a local folder of reference material").argument("<path>", "Folder path to index").option("--name <name>", "Store name (defaults to folder name)").action(async (path, options) => {
|
|
1398
912
|
await handleAddFolder({ path, ...options });
|
|
1399
913
|
});
|
|
1400
914
|
}
|
|
1401
915
|
function createStoresCommand(_getOptions) {
|
|
1402
|
-
return new
|
|
916
|
+
return new Command4("stores").description("List all indexed library stores").action(async () => {
|
|
1403
917
|
await handleStores();
|
|
1404
918
|
});
|
|
1405
919
|
}
|
|
1406
920
|
function createSuggestCommand(_getOptions) {
|
|
1407
|
-
return new
|
|
921
|
+
return new Command4("suggest").description("Suggest important dependencies to add to knowledge stores").action(async () => {
|
|
1408
922
|
await handleSuggest();
|
|
1409
923
|
});
|
|
1410
924
|
}
|
|
1411
925
|
|
|
926
|
+
// src/cli/commands/search.ts
|
|
927
|
+
import { Command as Command5 } from "commander";
|
|
928
|
+
function createSearchCommand(getOptions) {
|
|
929
|
+
const search = new Command5("search").description("Search indexed documents using vector similarity + full-text matching").argument("<query>", "Search query").option(
|
|
930
|
+
"-s, --stores <stores>",
|
|
931
|
+
"Limit search to specific stores (comma-separated IDs or names)"
|
|
932
|
+
).option(
|
|
933
|
+
"-m, --mode <mode>",
|
|
934
|
+
"vector (embeddings only), fts (text only), hybrid (both, default)",
|
|
935
|
+
"hybrid"
|
|
936
|
+
).option("-n, --limit <count>", "Maximum results to return (default: 10)", "10").option("-t, --threshold <score>", "Minimum score 0-1; omit low-relevance results").option("--include-content", "Show full document content, not just preview snippet").option(
|
|
937
|
+
"--detail <level>",
|
|
938
|
+
"Context detail: minimal, contextual, full (default: minimal)",
|
|
939
|
+
"minimal"
|
|
940
|
+
).action(
|
|
941
|
+
async (query, options) => {
|
|
942
|
+
const globalOpts = getOptions();
|
|
943
|
+
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
944
|
+
let exitCode = 0;
|
|
945
|
+
try {
|
|
946
|
+
let storeIds = (await services.store.list()).map((s) => s.id);
|
|
947
|
+
searchLogic: {
|
|
948
|
+
if (options.stores !== void 0) {
|
|
949
|
+
const requestedStores = options.stores.split(",").map((s) => s.trim());
|
|
950
|
+
const resolvedStores = [];
|
|
951
|
+
for (const requested of requestedStores) {
|
|
952
|
+
const store = await services.store.getByIdOrName(requested);
|
|
953
|
+
if (store !== void 0) {
|
|
954
|
+
resolvedStores.push(store.id);
|
|
955
|
+
} else {
|
|
956
|
+
console.error(`Error: Store not found: ${requested}`);
|
|
957
|
+
exitCode = 3;
|
|
958
|
+
break searchLogic;
|
|
959
|
+
}
|
|
960
|
+
}
|
|
961
|
+
storeIds = resolvedStores;
|
|
962
|
+
}
|
|
963
|
+
if (storeIds.length === 0) {
|
|
964
|
+
console.error("No stores to search. Create a store first.");
|
|
965
|
+
exitCode = 1;
|
|
966
|
+
break searchLogic;
|
|
967
|
+
}
|
|
968
|
+
for (const storeId of storeIds) {
|
|
969
|
+
await services.lance.initialize(storeId);
|
|
970
|
+
}
|
|
971
|
+
const results = await services.search.search({
|
|
972
|
+
query,
|
|
973
|
+
stores: storeIds,
|
|
974
|
+
mode: options.mode ?? "hybrid",
|
|
975
|
+
limit: parseInt(options.limit ?? "10", 10),
|
|
976
|
+
threshold: options.threshold !== void 0 ? parseFloat(options.threshold) : void 0,
|
|
977
|
+
includeContent: options.includeContent,
|
|
978
|
+
detail: options.detail ?? "minimal"
|
|
979
|
+
});
|
|
980
|
+
if (globalOpts.format === "json") {
|
|
981
|
+
console.log(JSON.stringify(results, null, 2));
|
|
982
|
+
} else if (globalOpts.quiet === true) {
|
|
983
|
+
for (const r of results.results) {
|
|
984
|
+
const path = r.metadata.path ?? r.metadata.url ?? "unknown";
|
|
985
|
+
console.log(path);
|
|
986
|
+
}
|
|
987
|
+
} else {
|
|
988
|
+
console.log(`
|
|
989
|
+
Search: "${query}"`);
|
|
990
|
+
console.log(
|
|
991
|
+
`Mode: ${results.mode} | Detail: ${String(options.detail)} | Stores: ${String(results.stores.length)} | Results: ${String(results.totalResults)} | Time: ${String(results.timeMs)}ms
|
|
992
|
+
`
|
|
993
|
+
);
|
|
994
|
+
if (results.results.length === 0) {
|
|
995
|
+
console.log("No results found.\n");
|
|
996
|
+
} else {
|
|
997
|
+
for (let i = 0; i < results.results.length; i++) {
|
|
998
|
+
const r = results.results[i];
|
|
999
|
+
if (r === void 0) continue;
|
|
1000
|
+
if (r.summary) {
|
|
1001
|
+
console.log(
|
|
1002
|
+
`${String(i + 1)}. [${r.score.toFixed(2)}] ${r.summary.type}: ${r.summary.name}`
|
|
1003
|
+
);
|
|
1004
|
+
console.log(` ${r.summary.location}`);
|
|
1005
|
+
console.log(` ${r.summary.purpose}`);
|
|
1006
|
+
if (r.context && options.detail !== "minimal") {
|
|
1007
|
+
console.log(` Imports: ${r.context.keyImports.slice(0, 3).join(", ")}`);
|
|
1008
|
+
console.log(
|
|
1009
|
+
` Related: ${r.context.relatedConcepts.slice(0, 3).join(", ")}`
|
|
1010
|
+
);
|
|
1011
|
+
}
|
|
1012
|
+
console.log();
|
|
1013
|
+
} else {
|
|
1014
|
+
const path = r.metadata.path ?? r.metadata.url ?? "unknown";
|
|
1015
|
+
console.log(`${String(i + 1)}. [${r.score.toFixed(2)}] ${path}`);
|
|
1016
|
+
const preview = r.highlight ?? r.content.slice(0, 150).replace(/\n/g, " ") + (r.content.length > 150 ? "..." : "");
|
|
1017
|
+
console.log(` ${preview}
|
|
1018
|
+
`);
|
|
1019
|
+
}
|
|
1020
|
+
}
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
}
|
|
1024
|
+
} finally {
|
|
1025
|
+
await destroyServices(services);
|
|
1026
|
+
}
|
|
1027
|
+
if (exitCode !== 0) {
|
|
1028
|
+
process.exit(exitCode);
|
|
1029
|
+
}
|
|
1030
|
+
}
|
|
1031
|
+
);
|
|
1032
|
+
return search;
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
// src/cli/commands/serve.ts
|
|
1036
|
+
import { serve } from "@hono/node-server";
|
|
1037
|
+
import { Command as Command6 } from "commander";
|
|
1038
|
+
|
|
1039
|
+
// src/server/app.ts
|
|
1040
|
+
import { Hono } from "hono";
|
|
1041
|
+
import { cors } from "hono/cors";
|
|
1042
|
+
import { z } from "zod";
|
|
1043
|
+
var CreateStoreBodySchema = z.object({
|
|
1044
|
+
name: z.string().min(1, "Store name must be a non-empty string"),
|
|
1045
|
+
type: z.enum(["file", "repo", "web"]),
|
|
1046
|
+
path: z.string().min(1).optional(),
|
|
1047
|
+
url: z.string().min(1).optional(),
|
|
1048
|
+
description: z.string().optional(),
|
|
1049
|
+
tags: z.array(z.string()).optional(),
|
|
1050
|
+
branch: z.string().optional(),
|
|
1051
|
+
depth: z.number().int().positive().optional()
|
|
1052
|
+
}).refine(
|
|
1053
|
+
(data) => {
|
|
1054
|
+
switch (data.type) {
|
|
1055
|
+
case "file":
|
|
1056
|
+
return data.path !== void 0;
|
|
1057
|
+
case "web":
|
|
1058
|
+
return data.url !== void 0;
|
|
1059
|
+
case "repo":
|
|
1060
|
+
return data.path !== void 0 || data.url !== void 0;
|
|
1061
|
+
}
|
|
1062
|
+
},
|
|
1063
|
+
{
|
|
1064
|
+
message: "Missing required field: file stores need path, web stores need url, repo stores need path or url"
|
|
1065
|
+
}
|
|
1066
|
+
);
|
|
1067
|
+
var SearchBodySchema = z.object({
|
|
1068
|
+
query: z.string().min(1, "Query must be a non-empty string"),
|
|
1069
|
+
detail: z.enum(["minimal", "contextual", "full"]).optional(),
|
|
1070
|
+
limit: z.number().int().positive().optional(),
|
|
1071
|
+
stores: z.array(z.string()).optional()
|
|
1072
|
+
});
|
|
1073
|
+
function createApp(services) {
|
|
1074
|
+
const app = new Hono();
|
|
1075
|
+
app.use("*", cors());
|
|
1076
|
+
app.get("/health", (c) => c.json({ status: "ok" }));
|
|
1077
|
+
app.get("/api/stores", async (c) => {
|
|
1078
|
+
const stores = await services.store.list();
|
|
1079
|
+
return c.json(stores);
|
|
1080
|
+
});
|
|
1081
|
+
app.post("/api/stores", async (c) => {
|
|
1082
|
+
const jsonData = await c.req.json();
|
|
1083
|
+
const parseResult = CreateStoreBodySchema.safeParse(jsonData);
|
|
1084
|
+
if (!parseResult.success) {
|
|
1085
|
+
return c.json({ error: parseResult.error.issues[0]?.message ?? "Invalid request body" }, 400);
|
|
1086
|
+
}
|
|
1087
|
+
const result = await services.store.create(parseResult.data);
|
|
1088
|
+
if (result.success) {
|
|
1089
|
+
return c.json(result.data, 201);
|
|
1090
|
+
}
|
|
1091
|
+
return c.json({ error: result.error.message }, 400);
|
|
1092
|
+
});
|
|
1093
|
+
app.get("/api/stores/:id", async (c) => {
|
|
1094
|
+
const store = await services.store.getByIdOrName(c.req.param("id"));
|
|
1095
|
+
if (!store) return c.json({ error: "Not found" }, 404);
|
|
1096
|
+
return c.json(store);
|
|
1097
|
+
});
|
|
1098
|
+
app.delete("/api/stores/:id", async (c) => {
|
|
1099
|
+
const store = await services.store.getByIdOrName(c.req.param("id"));
|
|
1100
|
+
if (!store) return c.json({ error: "Not found" }, 404);
|
|
1101
|
+
const result = await services.store.delete(store.id);
|
|
1102
|
+
if (result.success) return c.json({ deleted: true });
|
|
1103
|
+
return c.json({ error: result.error.message }, 400);
|
|
1104
|
+
});
|
|
1105
|
+
app.post("/api/search", async (c) => {
|
|
1106
|
+
const jsonData = await c.req.json();
|
|
1107
|
+
const parseResult = SearchBodySchema.safeParse(jsonData);
|
|
1108
|
+
if (!parseResult.success) {
|
|
1109
|
+
return c.json({ error: parseResult.error.issues[0]?.message ?? "Invalid request body" }, 400);
|
|
1110
|
+
}
|
|
1111
|
+
const storeIds = (await services.store.list()).map((s) => s.id);
|
|
1112
|
+
for (const id of storeIds) {
|
|
1113
|
+
await services.lance.initialize(id);
|
|
1114
|
+
}
|
|
1115
|
+
const requestedStores = parseResult.data.stores !== void 0 ? parseResult.data.stores.map((s) => createStoreId(s)) : storeIds;
|
|
1116
|
+
const query = {
|
|
1117
|
+
query: parseResult.data.query,
|
|
1118
|
+
detail: parseResult.data.detail ?? "minimal",
|
|
1119
|
+
limit: parseResult.data.limit ?? 10,
|
|
1120
|
+
stores: requestedStores
|
|
1121
|
+
};
|
|
1122
|
+
const results = await services.search.search(query);
|
|
1123
|
+
return c.json(results);
|
|
1124
|
+
});
|
|
1125
|
+
app.post("/api/stores/:id/index", async (c) => {
|
|
1126
|
+
const store = await services.store.getByIdOrName(c.req.param("id"));
|
|
1127
|
+
if (!store) return c.json({ error: "Not found" }, 404);
|
|
1128
|
+
await services.lance.initialize(store.id);
|
|
1129
|
+
const result = await services.index.indexStore(store);
|
|
1130
|
+
if (result.success) return c.json(result.data);
|
|
1131
|
+
return c.json({ error: result.error.message }, 400);
|
|
1132
|
+
});
|
|
1133
|
+
return app;
|
|
1134
|
+
}
|
|
1135
|
+
|
|
1136
|
+
// src/cli/commands/serve.ts
|
|
1137
|
+
function createServeCommand(getOptions) {
|
|
1138
|
+
return new Command6("serve").description("Start HTTP API server for programmatic search access").option("-p, --port <port>", "Port to listen on (default: 3847)", "3847").option(
|
|
1139
|
+
"--host <host>",
|
|
1140
|
+
"Bind address (default: 127.0.0.1, use 0.0.0.0 for all interfaces)",
|
|
1141
|
+
"127.0.0.1"
|
|
1142
|
+
).action(async (options) => {
|
|
1143
|
+
const globalOpts = getOptions();
|
|
1144
|
+
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
1145
|
+
const app = createApp(services);
|
|
1146
|
+
const port = parseInt(options.port ?? "3847", 10);
|
|
1147
|
+
const host = options.host ?? "127.0.0.1";
|
|
1148
|
+
const shutdown = () => {
|
|
1149
|
+
void (async () => {
|
|
1150
|
+
await destroyServices(services);
|
|
1151
|
+
process.exit(0);
|
|
1152
|
+
})();
|
|
1153
|
+
};
|
|
1154
|
+
process.on("SIGINT", shutdown);
|
|
1155
|
+
process.on("SIGTERM", shutdown);
|
|
1156
|
+
console.log(`Starting server on http://${host}:${String(port)}`);
|
|
1157
|
+
serve({
|
|
1158
|
+
fetch: app.fetch,
|
|
1159
|
+
port,
|
|
1160
|
+
hostname: host
|
|
1161
|
+
});
|
|
1162
|
+
});
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1165
|
+
// src/cli/commands/setup.ts
|
|
1166
|
+
import { spawnSync } from "child_process";
|
|
1167
|
+
import { existsSync as existsSync2 } from "fs";
|
|
1168
|
+
import { mkdir } from "fs/promises";
|
|
1169
|
+
import { homedir } from "os";
|
|
1170
|
+
import { join as join2 } from "path";
|
|
1171
|
+
import { Command as Command7 } from "commander";
|
|
1172
|
+
import ora4 from "ora";
|
|
1173
|
+
|
|
1174
|
+
// src/defaults/repos.ts
|
|
1175
|
+
var DEFAULT_REPOS = [
|
|
1176
|
+
{
|
|
1177
|
+
url: "git@github.com:ericbuess/claude-code-docs.git",
|
|
1178
|
+
name: "claude-code-docs",
|
|
1179
|
+
description: "Claude Code documentation",
|
|
1180
|
+
tags: ["claude", "docs", "claude-code"]
|
|
1181
|
+
},
|
|
1182
|
+
{
|
|
1183
|
+
url: "git@github.com:anthropics/claude-code.git",
|
|
1184
|
+
name: "claude-code",
|
|
1185
|
+
description: "Claude Code CLI tool source",
|
|
1186
|
+
tags: ["claude", "cli", "anthropic"]
|
|
1187
|
+
},
|
|
1188
|
+
{
|
|
1189
|
+
url: "git@github.com:anthropics/claude-agent-sdk-python.git",
|
|
1190
|
+
name: "claude-agent-sdk-python",
|
|
1191
|
+
description: "Claude Agent SDK for Python",
|
|
1192
|
+
tags: ["claude", "sdk", "python", "agents"]
|
|
1193
|
+
},
|
|
1194
|
+
{
|
|
1195
|
+
url: "git@github.com:anthropics/skills.git",
|
|
1196
|
+
name: "claude-skills",
|
|
1197
|
+
description: "Claude skills and capabilities",
|
|
1198
|
+
tags: ["claude", "skills"]
|
|
1199
|
+
},
|
|
1200
|
+
{
|
|
1201
|
+
url: "git@github.com:anthropics/claude-quickstarts.git",
|
|
1202
|
+
name: "claude-quickstarts",
|
|
1203
|
+
description: "Claude quickstart examples and tutorials",
|
|
1204
|
+
tags: ["claude", "examples", "tutorials"]
|
|
1205
|
+
},
|
|
1206
|
+
{
|
|
1207
|
+
url: "git@github.com:anthropics/claude-plugins-official.git",
|
|
1208
|
+
name: "claude-plugins",
|
|
1209
|
+
description: "Official Claude plugins",
|
|
1210
|
+
tags: ["claude", "plugins"]
|
|
1211
|
+
},
|
|
1212
|
+
{
|
|
1213
|
+
url: "git@github.com:anthropics/claude-agent-sdk-typescript.git",
|
|
1214
|
+
name: "claude-agent-sdk-typescript",
|
|
1215
|
+
description: "Claude Agent SDK for TypeScript",
|
|
1216
|
+
tags: ["claude", "sdk", "typescript", "agents"]
|
|
1217
|
+
},
|
|
1218
|
+
{
|
|
1219
|
+
url: "git@github.com:anthropics/claude-agent-sdk-demos.git",
|
|
1220
|
+
name: "claude-agent-sdk-demos",
|
|
1221
|
+
description: "Claude Agent SDK demo applications",
|
|
1222
|
+
tags: ["claude", "sdk", "demos", "examples"]
|
|
1223
|
+
}
|
|
1224
|
+
];
|
|
1225
|
+
|
|
1226
|
+
// src/cli/commands/setup.ts
|
|
1227
|
+
var DEFAULT_REPOS_DIR = join2(homedir(), ".bluera", "bluera-knowledge", "repos");
|
|
1228
|
+
function createSetupCommand(getOptions) {
|
|
1229
|
+
const setup = new Command7("setup").description(
|
|
1230
|
+
"Quick-start with pre-configured Claude/Anthropic documentation repos"
|
|
1231
|
+
);
|
|
1232
|
+
setup.command("repos").description(
|
|
1233
|
+
"Clone repos to ~/.bluera/bluera-knowledge/repos/, create stores, index all content"
|
|
1234
|
+
).option(
|
|
1235
|
+
"--repos-dir <path>",
|
|
1236
|
+
"Clone destination (default: ~/.bluera/bluera-knowledge/repos/)",
|
|
1237
|
+
DEFAULT_REPOS_DIR
|
|
1238
|
+
).option("--skip-clone", "Don't clone; assume repos already exist locally").option("--skip-index", "Clone and create stores but don't index yet").option("--only <names>", "Only process matching repos (comma-separated, partial match)").option("--list", "Print available repos without cloning/indexing").action(
|
|
1239
|
+
async (options) => {
|
|
1240
|
+
const globalOpts = getOptions();
|
|
1241
|
+
if (options.list === true) {
|
|
1242
|
+
console.log("\nDefault repositories:\n");
|
|
1243
|
+
for (const repo of DEFAULT_REPOS) {
|
|
1244
|
+
console.log(` ${repo.name}`);
|
|
1245
|
+
console.log(` URL: ${repo.url}`);
|
|
1246
|
+
console.log(` Description: ${repo.description}`);
|
|
1247
|
+
console.log(` Tags: ${repo.tags.join(", ")}`);
|
|
1248
|
+
console.log("");
|
|
1249
|
+
}
|
|
1250
|
+
return;
|
|
1251
|
+
}
|
|
1252
|
+
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
1253
|
+
try {
|
|
1254
|
+
let repos = DEFAULT_REPOS;
|
|
1255
|
+
if (options.only !== void 0 && options.only !== "") {
|
|
1256
|
+
const onlyNames = options.only.split(",").map((n) => n.trim().toLowerCase());
|
|
1257
|
+
repos = DEFAULT_REPOS.filter(
|
|
1258
|
+
(r) => onlyNames.some((n) => r.name.toLowerCase().includes(n))
|
|
1259
|
+
);
|
|
1260
|
+
if (repos.length === 0) {
|
|
1261
|
+
console.error(`No repos matched: ${options.only}`);
|
|
1262
|
+
console.log("Available repos:", DEFAULT_REPOS.map((r) => r.name).join(", "));
|
|
1263
|
+
process.exit(1);
|
|
1264
|
+
}
|
|
1265
|
+
}
|
|
1266
|
+
console.log(`
|
|
1267
|
+
Setting up ${String(repos.length)} repositories...
|
|
1268
|
+
`);
|
|
1269
|
+
await mkdir(options.reposDir, { recursive: true });
|
|
1270
|
+
for (const repo of repos) {
|
|
1271
|
+
const repoPath = join2(options.reposDir, repo.name);
|
|
1272
|
+
const spinner = ora4(`Processing ${repo.name}`).start();
|
|
1273
|
+
try {
|
|
1274
|
+
if (options.skipClone !== true) {
|
|
1275
|
+
if (existsSync2(repoPath)) {
|
|
1276
|
+
spinner.text = `${repo.name}: Already cloned, pulling latest...`;
|
|
1277
|
+
const pullResult = spawnSync("git", ["pull", "--ff-only"], {
|
|
1278
|
+
cwd: repoPath,
|
|
1279
|
+
stdio: "pipe"
|
|
1280
|
+
});
|
|
1281
|
+
if (pullResult.status !== 0) {
|
|
1282
|
+
spinner.text = `${repo.name}: Pull skipped (local changes)`;
|
|
1283
|
+
}
|
|
1284
|
+
} else {
|
|
1285
|
+
spinner.text = `${repo.name}: Cloning...`;
|
|
1286
|
+
const cloneResult = spawnSync("git", ["clone", repo.url, repoPath], {
|
|
1287
|
+
stdio: "pipe"
|
|
1288
|
+
});
|
|
1289
|
+
if (cloneResult.status !== 0) {
|
|
1290
|
+
const errorMessage = cloneResult.stderr.length > 0 ? cloneResult.stderr.toString() : "Git clone failed";
|
|
1291
|
+
throw new Error(errorMessage);
|
|
1292
|
+
}
|
|
1293
|
+
}
|
|
1294
|
+
}
|
|
1295
|
+
spinner.text = `${repo.name}: Creating store...`;
|
|
1296
|
+
const existingStore = await services.store.getByIdOrName(repo.name);
|
|
1297
|
+
let storeId;
|
|
1298
|
+
if (existingStore) {
|
|
1299
|
+
storeId = existingStore.id;
|
|
1300
|
+
spinner.text = `${repo.name}: Store already exists`;
|
|
1301
|
+
} else {
|
|
1302
|
+
const result = await services.store.create({
|
|
1303
|
+
name: repo.name,
|
|
1304
|
+
type: "repo",
|
|
1305
|
+
path: repoPath,
|
|
1306
|
+
description: repo.description,
|
|
1307
|
+
tags: repo.tags
|
|
1308
|
+
});
|
|
1309
|
+
if (!result.success) {
|
|
1310
|
+
throw new Error(
|
|
1311
|
+
result.error instanceof Error ? result.error.message : String(result.error)
|
|
1312
|
+
);
|
|
1313
|
+
}
|
|
1314
|
+
storeId = result.data.id;
|
|
1315
|
+
}
|
|
1316
|
+
if (options.skipIndex !== true) {
|
|
1317
|
+
spinner.text = `${repo.name}: Indexing...`;
|
|
1318
|
+
const store = await services.store.getByIdOrName(storeId);
|
|
1319
|
+
if (store) {
|
|
1320
|
+
await services.lance.initialize(store.id);
|
|
1321
|
+
const indexResult = await services.index.indexStore(store, (event) => {
|
|
1322
|
+
if (event.type === "progress") {
|
|
1323
|
+
spinner.text = `${repo.name}: Indexing ${String(event.current)}/${String(event.total)} files`;
|
|
1324
|
+
}
|
|
1325
|
+
});
|
|
1326
|
+
if (indexResult.success) {
|
|
1327
|
+
spinner.succeed(
|
|
1328
|
+
`${repo.name}: ${String(indexResult.data.documentsIndexed)} docs, ${String(indexResult.data.chunksCreated)} chunks`
|
|
1329
|
+
);
|
|
1330
|
+
} else {
|
|
1331
|
+
throw new Error(
|
|
1332
|
+
indexResult.error instanceof Error ? indexResult.error.message : String(indexResult.error)
|
|
1333
|
+
);
|
|
1334
|
+
}
|
|
1335
|
+
}
|
|
1336
|
+
} else {
|
|
1337
|
+
spinner.succeed(`${repo.name}: Ready (indexing skipped)`);
|
|
1338
|
+
}
|
|
1339
|
+
} catch (error) {
|
|
1340
|
+
spinner.fail(
|
|
1341
|
+
`${repo.name}: ${error instanceof Error ? error.message : String(error)}`
|
|
1342
|
+
);
|
|
1343
|
+
}
|
|
1344
|
+
}
|
|
1345
|
+
console.log('\nSetup complete! Use "bluera-knowledge search <query>" to search.\n');
|
|
1346
|
+
} finally {
|
|
1347
|
+
await destroyServices(services);
|
|
1348
|
+
}
|
|
1349
|
+
}
|
|
1350
|
+
);
|
|
1351
|
+
return setup;
|
|
1352
|
+
}
|
|
1353
|
+
|
|
1354
|
+
// src/cli/commands/store.ts
|
|
1355
|
+
import { Command as Command8 } from "commander";
|
|
1356
|
+
function createStoreCommand(getOptions) {
|
|
1357
|
+
const store = new Command8("store").description(
|
|
1358
|
+
"Manage knowledge stores (collections of indexed documents)"
|
|
1359
|
+
);
|
|
1360
|
+
store.command("list").description("Show all stores with their type (file/repo/web) and ID").option("-t, --type <type>", "Filter by type: file, repo, or web").action(async (options) => {
|
|
1361
|
+
const globalOpts = getOptions();
|
|
1362
|
+
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
1363
|
+
try {
|
|
1364
|
+
const stores = await services.store.list(options.type);
|
|
1365
|
+
if (globalOpts.format === "json") {
|
|
1366
|
+
console.log(JSON.stringify(stores, null, 2));
|
|
1367
|
+
} else if (globalOpts.quiet === true) {
|
|
1368
|
+
for (const s of stores) {
|
|
1369
|
+
console.log(s.name);
|
|
1370
|
+
}
|
|
1371
|
+
} else {
|
|
1372
|
+
if (stores.length === 0) {
|
|
1373
|
+
console.log("No stores found.");
|
|
1374
|
+
} else {
|
|
1375
|
+
console.log("\nStores:\n");
|
|
1376
|
+
for (const s of stores) {
|
|
1377
|
+
console.log(` ${s.name} (${s.type}) - ${s.id}`);
|
|
1378
|
+
}
|
|
1379
|
+
console.log("");
|
|
1380
|
+
}
|
|
1381
|
+
}
|
|
1382
|
+
} finally {
|
|
1383
|
+
await destroyServices(services);
|
|
1384
|
+
}
|
|
1385
|
+
});
|
|
1386
|
+
store.command("create <name>").description("Create a new store pointing to a local path or URL").requiredOption(
|
|
1387
|
+
"-t, --type <type>",
|
|
1388
|
+
"Store type: file (local dir), repo (git), web (crawled site)"
|
|
1389
|
+
).requiredOption("-s, --source <path>", "Local path for file/repo stores, URL for web stores").option("-d, --description <desc>", "Optional description for the store").option("--tags <tags>", "Comma-separated tags for filtering").action(
|
|
1390
|
+
async (name, options) => {
|
|
1391
|
+
const globalOpts = getOptions();
|
|
1392
|
+
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
1393
|
+
let exitCode = 0;
|
|
1394
|
+
try {
|
|
1395
|
+
const isUrl = options.source.startsWith("http://") || options.source.startsWith("https://");
|
|
1396
|
+
const result = await services.store.create({
|
|
1397
|
+
name,
|
|
1398
|
+
type: options.type,
|
|
1399
|
+
path: options.type === "file" || options.type === "repo" && !isUrl ? options.source : void 0,
|
|
1400
|
+
url: options.type === "web" || options.type === "repo" && isUrl ? options.source : void 0,
|
|
1401
|
+
description: options.description,
|
|
1402
|
+
tags: options.tags?.split(",").map((t) => t.trim())
|
|
1403
|
+
});
|
|
1404
|
+
if (result.success) {
|
|
1405
|
+
if (globalOpts.format === "json") {
|
|
1406
|
+
console.log(JSON.stringify(result.data, null, 2));
|
|
1407
|
+
} else {
|
|
1408
|
+
console.log(`
|
|
1409
|
+
Created store: ${result.data.name} (${result.data.id})
|
|
1410
|
+
`);
|
|
1411
|
+
}
|
|
1412
|
+
} else {
|
|
1413
|
+
console.error(`Error: ${result.error.message}`);
|
|
1414
|
+
exitCode = 1;
|
|
1415
|
+
}
|
|
1416
|
+
} finally {
|
|
1417
|
+
await destroyServices(services);
|
|
1418
|
+
}
|
|
1419
|
+
if (exitCode !== 0) {
|
|
1420
|
+
process.exit(exitCode);
|
|
1421
|
+
}
|
|
1422
|
+
}
|
|
1423
|
+
);
|
|
1424
|
+
store.command("info <store>").description("Show store details: ID, type, path/URL, timestamps").action(async (storeIdOrName) => {
|
|
1425
|
+
const globalOpts = getOptions();
|
|
1426
|
+
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
1427
|
+
try {
|
|
1428
|
+
const s = await services.store.getByIdOrName(storeIdOrName);
|
|
1429
|
+
if (s === void 0) {
|
|
1430
|
+
console.error(`Error: Store not found: ${storeIdOrName}`);
|
|
1431
|
+
process.exit(3);
|
|
1432
|
+
}
|
|
1433
|
+
if (globalOpts.format === "json") {
|
|
1434
|
+
console.log(JSON.stringify(s, null, 2));
|
|
1435
|
+
} else {
|
|
1436
|
+
console.log(`
|
|
1437
|
+
Store: ${s.name}`);
|
|
1438
|
+
console.log(` ID: ${s.id}`);
|
|
1439
|
+
console.log(` Type: ${s.type}`);
|
|
1440
|
+
if ("path" in s) console.log(` Path: ${s.path}`);
|
|
1441
|
+
if ("url" in s && s.url !== void 0) console.log(` URL: ${s.url}`);
|
|
1442
|
+
if (s.description !== void 0) console.log(` Description: ${s.description}`);
|
|
1443
|
+
console.log(` Created: ${s.createdAt.toISOString()}`);
|
|
1444
|
+
console.log(` Updated: ${s.updatedAt.toISOString()}`);
|
|
1445
|
+
console.log("");
|
|
1446
|
+
}
|
|
1447
|
+
} finally {
|
|
1448
|
+
await destroyServices(services);
|
|
1449
|
+
}
|
|
1450
|
+
});
|
|
1451
|
+
store.command("delete <store>").description("Remove store and its indexed documents from LanceDB").option("-f, --force", "Delete without confirmation prompt").option("-y, --yes", "Alias for --force").action(async (storeIdOrName, options) => {
|
|
1452
|
+
const globalOpts = getOptions();
|
|
1453
|
+
const services = await createServices(globalOpts.config, globalOpts.dataDir);
|
|
1454
|
+
try {
|
|
1455
|
+
const s = await services.store.getByIdOrName(storeIdOrName);
|
|
1456
|
+
if (s === void 0) {
|
|
1457
|
+
console.error(`Error: Store not found: ${storeIdOrName}`);
|
|
1458
|
+
process.exit(3);
|
|
1459
|
+
}
|
|
1460
|
+
const skipConfirmation = options.force === true || options.yes === true;
|
|
1461
|
+
if (!skipConfirmation) {
|
|
1462
|
+
if (!process.stdin.isTTY) {
|
|
1463
|
+
console.error(
|
|
1464
|
+
"Error: Use --force or -y to delete without confirmation in non-interactive mode"
|
|
1465
|
+
);
|
|
1466
|
+
process.exit(1);
|
|
1467
|
+
}
|
|
1468
|
+
const readline = await import("readline");
|
|
1469
|
+
const rl = readline.createInterface({
|
|
1470
|
+
input: process.stdin,
|
|
1471
|
+
output: process.stdout
|
|
1472
|
+
});
|
|
1473
|
+
const answer = await new Promise((resolve) => {
|
|
1474
|
+
rl.question(`Delete store "${s.name}"? [y/N] `, resolve);
|
|
1475
|
+
});
|
|
1476
|
+
rl.close();
|
|
1477
|
+
if (answer.toLowerCase() !== "y" && answer.toLowerCase() !== "yes") {
|
|
1478
|
+
console.log("Cancelled.");
|
|
1479
|
+
process.exit(0);
|
|
1480
|
+
}
|
|
1481
|
+
}
|
|
1482
|
+
const result = await services.store.delete(s.id);
|
|
1483
|
+
if (result.success) {
|
|
1484
|
+
console.log(`Deleted store: ${s.name}`);
|
|
1485
|
+
} else {
|
|
1486
|
+
console.error(`Error: ${result.error.message}`);
|
|
1487
|
+
process.exit(1);
|
|
1488
|
+
}
|
|
1489
|
+
} finally {
|
|
1490
|
+
await destroyServices(services);
|
|
1491
|
+
}
|
|
1492
|
+
});
|
|
1493
|
+
return store;
|
|
1494
|
+
}
|
|
1495
|
+
|
|
1496
|
+
// src/cli/program.ts
|
|
1497
|
+
import { readFileSync } from "fs";
|
|
1498
|
+
import { dirname, join as join3 } from "path";
|
|
1499
|
+
import { fileURLToPath } from "url";
|
|
1500
|
+
import { Command as Command9 } from "commander";
|
|
1501
|
+
function getVersion() {
|
|
1502
|
+
const __filename2 = fileURLToPath(import.meta.url);
|
|
1503
|
+
const __dirname2 = dirname(__filename2);
|
|
1504
|
+
const content = readFileSync(join3(__dirname2, "../package.json"), "utf-8");
|
|
1505
|
+
const pkg = JSON.parse(content);
|
|
1506
|
+
return pkg.version;
|
|
1507
|
+
}
|
|
1508
|
+
var version = getVersion();
|
|
1509
|
+
function createProgram() {
|
|
1510
|
+
const program2 = new Command9();
|
|
1511
|
+
program2.name("bluera-knowledge").description("CLI tool for managing knowledge stores with semantic search").version(version);
|
|
1512
|
+
program2.option("-c, --config <path>", "Path to config file").option("-d, --data-dir <path>", "Data directory").option("-p, --project-root <path>", "Project root directory (for resolving relative paths)").option("-f, --format <format>", "Output format: json | table | plain", "table").option("-q, --quiet", "Suppress non-essential output").option("-v, --verbose", "Enable verbose logging");
|
|
1513
|
+
return program2;
|
|
1514
|
+
}
|
|
1515
|
+
function getGlobalOptions(program2) {
|
|
1516
|
+
const opts = program2.opts();
|
|
1517
|
+
return {
|
|
1518
|
+
config: opts.config,
|
|
1519
|
+
dataDir: opts.dataDir,
|
|
1520
|
+
projectRoot: opts.projectRoot,
|
|
1521
|
+
format: opts.format,
|
|
1522
|
+
quiet: opts.quiet,
|
|
1523
|
+
verbose: opts.verbose
|
|
1524
|
+
};
|
|
1525
|
+
}
|
|
1526
|
+
|
|
1412
1527
|
// src/index.ts
|
|
1413
1528
|
var DEFAULT_DATA_DIR = join4(homedir2(), ".bluera", "bluera-knowledge", "data");
|
|
1414
1529
|
var DEFAULT_CONFIG = join4(homedir2(), ".bluera", "bluera-knowledge", "config.json");
|
|
@@ -1421,7 +1536,7 @@ function formatCommandHelp(cmd, indent = "") {
|
|
|
1421
1536
|
const req = a.required;
|
|
1422
1537
|
return req ? `<${a.name()}>` : `[${a.name()}]`;
|
|
1423
1538
|
}).join(" ");
|
|
1424
|
-
lines.push(`${indent}${name}${args ?
|
|
1539
|
+
lines.push(`${indent}${name}${args ? ` ${args}` : ""}`);
|
|
1425
1540
|
if (desc) {
|
|
1426
1541
|
lines.push(`${indent} ${desc}`);
|
|
1427
1542
|
}
|
|
@@ -1432,7 +1547,7 @@ function formatCommandHelp(cmd, indent = "") {
|
|
|
1432
1547
|
const subcommands = cmd.commands.filter((c) => c.name() !== "help");
|
|
1433
1548
|
for (const sub of subcommands) {
|
|
1434
1549
|
lines.push("");
|
|
1435
|
-
lines.push(...formatCommandHelp(sub, indent
|
|
1550
|
+
lines.push(...formatCommandHelp(sub, `${indent} `));
|
|
1436
1551
|
}
|
|
1437
1552
|
return lines;
|
|
1438
1553
|
}
|
|
@@ -1443,7 +1558,9 @@ function printFullHelp(program2) {
|
|
|
1443
1558
|
console.log(` config ${DEFAULT_CONFIG}`);
|
|
1444
1559
|
console.log(` repos ${DEFAULT_REPOS_DIR2}`);
|
|
1445
1560
|
console.log("\nGlobal options:");
|
|
1446
|
-
const globalOpts = program2.options.filter(
|
|
1561
|
+
const globalOpts = program2.options.filter(
|
|
1562
|
+
(o) => o.flags !== "-h, --help" && o.flags !== "-V, --version"
|
|
1563
|
+
);
|
|
1447
1564
|
for (const opt of globalOpts) {
|
|
1448
1565
|
console.log(` ${opt.flags.padEnd(28)} ${opt.description}`);
|
|
1449
1566
|
}
|