bluera-knowledge 0.9.32 → 0.9.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/hooks/post-edit-check.sh +5 -3
- package/.claude/skills/atomic-commits/SKILL.md +3 -1
- package/.husky/pre-commit +3 -2
- package/.prettierrc +9 -0
- package/.versionrc.json +1 -1
- package/CHANGELOG.md +33 -0
- package/CLAUDE.md +6 -0
- package/README.md +25 -13
- package/bun.lock +277 -33
- package/dist/{chunk-L2YVNC63.js → chunk-6FHWC36B.js} +9 -1
- package/dist/chunk-6FHWC36B.js.map +1 -0
- package/dist/{chunk-RST4XGRL.js → chunk-DC7CGSGT.js} +288 -241
- package/dist/chunk-DC7CGSGT.js.map +1 -0
- package/dist/{chunk-6PBP5DVD.js → chunk-WFNPNAAP.js} +3212 -3054
- package/dist/chunk-WFNPNAAP.js.map +1 -0
- package/dist/{chunk-WT2DAEO7.js → chunk-Z2KKVH45.js} +548 -482
- package/dist/chunk-Z2KKVH45.js.map +1 -0
- package/dist/index.js +871 -758
- package/dist/index.js.map +1 -1
- package/dist/mcp/server.js +3 -3
- package/dist/watch.service-BJV3TI3F.js +7 -0
- package/dist/workers/background-worker-cli.js +46 -45
- package/dist/workers/background-worker-cli.js.map +1 -1
- package/eslint.config.js +43 -1
- package/package.json +18 -11
- package/plugin.json +8 -0
- package/python/requirements.txt +1 -1
- package/src/analysis/ast-parser.test.ts +12 -11
- package/src/analysis/ast-parser.ts +28 -22
- package/src/analysis/code-graph.test.ts +52 -62
- package/src/analysis/code-graph.ts +9 -13
- package/src/analysis/dependency-usage-analyzer.test.ts +91 -271
- package/src/analysis/dependency-usage-analyzer.ts +52 -24
- package/src/analysis/go-ast-parser.test.ts +22 -22
- package/src/analysis/go-ast-parser.ts +18 -25
- package/src/analysis/parser-factory.test.ts +9 -9
- package/src/analysis/parser-factory.ts +3 -3
- package/src/analysis/python-ast-parser.test.ts +27 -27
- package/src/analysis/python-ast-parser.ts +2 -2
- package/src/analysis/repo-url-resolver.test.ts +82 -82
- package/src/analysis/rust-ast-parser.test.ts +19 -19
- package/src/analysis/rust-ast-parser.ts +17 -27
- package/src/analysis/tree-sitter-parser.test.ts +3 -3
- package/src/analysis/tree-sitter-parser.ts +10 -16
- package/src/cli/commands/crawl.test.ts +40 -24
- package/src/cli/commands/crawl.ts +186 -166
- package/src/cli/commands/index-cmd.test.ts +90 -90
- package/src/cli/commands/index-cmd.ts +52 -36
- package/src/cli/commands/mcp.test.ts +6 -6
- package/src/cli/commands/mcp.ts +2 -2
- package/src/cli/commands/plugin-api.test.ts +16 -18
- package/src/cli/commands/plugin-api.ts +9 -6
- package/src/cli/commands/search.test.ts +16 -7
- package/src/cli/commands/search.ts +124 -87
- package/src/cli/commands/serve.test.ts +67 -25
- package/src/cli/commands/serve.ts +18 -3
- package/src/cli/commands/setup.test.ts +176 -101
- package/src/cli/commands/setup.ts +140 -117
- package/src/cli/commands/store.test.ts +82 -53
- package/src/cli/commands/store.ts +56 -37
- package/src/cli/program.ts +2 -2
- package/src/crawl/article-converter.test.ts +4 -1
- package/src/crawl/article-converter.ts +46 -31
- package/src/crawl/bridge.test.ts +240 -132
- package/src/crawl/bridge.ts +87 -30
- package/src/crawl/claude-client.test.ts +124 -56
- package/src/crawl/claude-client.ts +7 -15
- package/src/crawl/intelligent-crawler.test.ts +65 -22
- package/src/crawl/intelligent-crawler.ts +86 -53
- package/src/crawl/markdown-utils.ts +1 -4
- package/src/db/embeddings.ts +4 -6
- package/src/db/lance.test.ts +4 -4
- package/src/db/lance.ts +16 -12
- package/src/index.ts +26 -17
- package/src/logging/index.ts +1 -5
- package/src/logging/logger.ts +3 -5
- package/src/logging/payload.test.ts +1 -1
- package/src/logging/payload.ts +3 -5
- package/src/mcp/commands/index.ts +2 -2
- package/src/mcp/commands/job.commands.ts +12 -18
- package/src/mcp/commands/meta.commands.ts +13 -13
- package/src/mcp/commands/registry.ts +5 -8
- package/src/mcp/commands/store.commands.ts +19 -19
- package/src/mcp/handlers/execute.handler.test.ts +10 -10
- package/src/mcp/handlers/execute.handler.ts +4 -5
- package/src/mcp/handlers/index.ts +10 -14
- package/src/mcp/handlers/job.handler.test.ts +10 -10
- package/src/mcp/handlers/job.handler.ts +22 -25
- package/src/mcp/handlers/search.handler.test.ts +36 -65
- package/src/mcp/handlers/search.handler.ts +135 -104
- package/src/mcp/handlers/store.handler.test.ts +41 -52
- package/src/mcp/handlers/store.handler.ts +108 -88
- package/src/mcp/schemas/index.test.ts +73 -68
- package/src/mcp/schemas/index.ts +18 -12
- package/src/mcp/server.test.ts +1 -1
- package/src/mcp/server.ts +59 -46
- package/src/plugin/commands.test.ts +230 -95
- package/src/plugin/commands.ts +24 -25
- package/src/plugin/dependency-analyzer.test.ts +52 -52
- package/src/plugin/dependency-analyzer.ts +85 -22
- package/src/plugin/git-clone.test.ts +24 -13
- package/src/plugin/git-clone.ts +3 -7
- package/src/server/app.test.ts +109 -109
- package/src/server/app.ts +32 -23
- package/src/server/index.test.ts +64 -66
- package/src/services/chunking.service.test.ts +32 -32
- package/src/services/chunking.service.ts +16 -9
- package/src/services/code-graph.service.test.ts +30 -36
- package/src/services/code-graph.service.ts +24 -10
- package/src/services/code-unit.service.test.ts +55 -11
- package/src/services/code-unit.service.ts +85 -11
- package/src/services/config.service.test.ts +37 -18
- package/src/services/config.service.ts +30 -7
- package/src/services/index.service.test.ts +49 -18
- package/src/services/index.service.ts +98 -48
- package/src/services/index.ts +6 -9
- package/src/services/job.service.test.ts +22 -22
- package/src/services/job.service.ts +18 -18
- package/src/services/project-root.service.test.ts +1 -3
- package/src/services/search.service.test.ts +248 -120
- package/src/services/search.service.ts +286 -156
- package/src/services/services.test.ts +1 -1
- package/src/services/snippet.service.test.ts +14 -6
- package/src/services/snippet.service.ts +7 -5
- package/src/services/store.service.test.ts +68 -29
- package/src/services/store.service.ts +41 -12
- package/src/services/watch.service.test.ts +34 -14
- package/src/services/watch.service.ts +11 -1
- package/src/types/brands.test.ts +3 -1
- package/src/types/index.ts +2 -13
- package/src/types/search.ts +10 -8
- package/src/utils/type-guards.test.ts +20 -15
- package/src/utils/type-guards.ts +1 -1
- package/src/workers/background-worker-cli.ts +2 -2
- package/src/workers/background-worker.test.ts +54 -40
- package/src/workers/background-worker.ts +76 -60
- package/src/workers/spawn-worker.test.ts +22 -10
- package/src/workers/spawn-worker.ts +6 -6
- package/tests/analysis/ast-parser.test.ts +3 -3
- package/tests/analysis/code-graph.test.ts +5 -5
- package/tests/fixtures/code-snippets/api/error-handling.ts +4 -15
- package/tests/fixtures/code-snippets/api/rest-controller.ts +3 -9
- package/tests/fixtures/code-snippets/auth/jwt-auth.ts +5 -21
- package/tests/fixtures/code-snippets/auth/oauth-flow.ts +4 -4
- package/tests/fixtures/code-snippets/database/repository-pattern.ts +11 -3
- package/tests/fixtures/corpus/oss-repos/hono/src/adapter/aws-lambda/handler.ts +2 -2
- package/tests/fixtures/corpus/oss-repos/hono/src/adapter/cloudflare-pages/handler.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/adapter/cloudflare-workers/serve-static.ts +2 -2
- package/tests/fixtures/corpus/oss-repos/hono/src/client/client.ts +2 -2
- package/tests/fixtures/corpus/oss-repos/hono/src/client/types.ts +22 -20
- package/tests/fixtures/corpus/oss-repos/hono/src/context.ts +13 -10
- package/tests/fixtures/corpus/oss-repos/hono/src/helper/accepts/accepts.ts +10 -7
- package/tests/fixtures/corpus/oss-repos/hono/src/helper/adapter/index.ts +2 -2
- package/tests/fixtures/corpus/oss-repos/hono/src/helper/css/index.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/helper/factory/index.ts +16 -16
- package/tests/fixtures/corpus/oss-repos/hono/src/helper/ssg/ssg.ts +2 -2
- package/tests/fixtures/corpus/oss-repos/hono/src/hono-base.ts +3 -3
- package/tests/fixtures/corpus/oss-repos/hono/src/hono.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/jsx/dom/css.ts +2 -2
- package/tests/fixtures/corpus/oss-repos/hono/src/jsx/dom/intrinsic-element/components.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/jsx/dom/render.ts +7 -7
- package/tests/fixtures/corpus/oss-repos/hono/src/jsx/hooks/index.ts +3 -3
- package/tests/fixtures/corpus/oss-repos/hono/src/jsx/intrinsic-element/components.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/jsx/utils.ts +6 -6
- package/tests/fixtures/corpus/oss-repos/hono/src/middleware/jsx-renderer/index.ts +3 -3
- package/tests/fixtures/corpus/oss-repos/hono/src/middleware/serve-static/index.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/preset/quick.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/preset/tiny.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/router/pattern-router/router.ts +2 -2
- package/tests/fixtures/corpus/oss-repos/hono/src/router/reg-exp-router/node.ts +4 -4
- package/tests/fixtures/corpus/oss-repos/hono/src/router/reg-exp-router/router.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/router/trie-router/node.ts +1 -1
- package/tests/fixtures/corpus/oss-repos/hono/src/types.ts +166 -169
- package/tests/fixtures/corpus/oss-repos/hono/src/utils/body.ts +8 -8
- package/tests/fixtures/corpus/oss-repos/hono/src/utils/color.ts +3 -3
- package/tests/fixtures/corpus/oss-repos/hono/src/utils/cookie.ts +2 -2
- package/tests/fixtures/corpus/oss-repos/hono/src/utils/encode.ts +2 -2
- package/tests/fixtures/corpus/oss-repos/hono/src/utils/types.ts +30 -33
- package/tests/fixtures/corpus/oss-repos/hono/src/validator/validator.ts +2 -2
- package/tests/fixtures/test-server.ts +3 -2
- package/tests/helpers/performance-metrics.ts +8 -25
- package/tests/helpers/search-relevance.ts +14 -69
- package/tests/integration/cli-consistency.test.ts +5 -4
- package/tests/integration/python-bridge.test.ts +13 -3
- package/tests/mcp/server.test.ts +1 -1
- package/tests/services/code-unit.service.test.ts +48 -0
- package/tests/services/job.service.test.ts +124 -0
- package/tests/services/search.progressive-context.test.ts +2 -2
- package/.claude-plugin/plugin.json +0 -13
- package/dist/chunk-6PBP5DVD.js.map +0 -1
- package/dist/chunk-L2YVNC63.js.map +0 -1
- package/dist/chunk-RST4XGRL.js.map +0 -1
- package/dist/chunk-WT2DAEO7.js.map +0 -1
- package/dist/watch.service-YAIKKDCF.js +0 -7
- package/skills/atomic-commits/SKILL.md +0 -77
- /package/dist/{watch.service-YAIKKDCF.js.map → watch.service-BJV3TI3F.js.map} +0 -0
|
@@ -892,7 +892,8 @@ describe('IndexService - Hash and Metadata', () => {
|
|
|
892
892
|
});
|
|
893
893
|
|
|
894
894
|
it('indexes code with function names', async () => {
|
|
895
|
-
const code =
|
|
895
|
+
const code =
|
|
896
|
+
'export function hello() {\n return "world";\n}\n\nfunction goodbye() {\n return "bye";\n}';
|
|
896
897
|
await writeFile(join(testFilesDir, 'code.ts'), code);
|
|
897
898
|
|
|
898
899
|
const store: FileStore = {
|
|
@@ -1245,36 +1246,56 @@ describe('IndexService - Code Graph Integration', () => {
|
|
|
1245
1246
|
// Verify the source files passed to buildGraph
|
|
1246
1247
|
const sourceFiles = buildGraphMock.mock.calls[0][0] as Array<{ path: string; content: string }>;
|
|
1247
1248
|
expect(sourceFiles.length).toBe(2);
|
|
1248
|
-
expect(sourceFiles.some(f => f.path.endsWith('.js'))).toBe(true);
|
|
1249
|
-
expect(sourceFiles.some(f => f.path.endsWith('.jsx'))).toBe(true);
|
|
1249
|
+
expect(sourceFiles.some((f) => f.path.endsWith('.js'))).toBe(true);
|
|
1250
|
+
expect(sourceFiles.some((f) => f.path.endsWith('.jsx'))).toBe(true);
|
|
1250
1251
|
});
|
|
1251
1252
|
});
|
|
1252
1253
|
|
|
1253
1254
|
describe('classifyWebContentType', () => {
|
|
1254
1255
|
describe('documentation-primary classification', () => {
|
|
1255
1256
|
it('classifies API reference URLs', () => {
|
|
1256
|
-
expect(classifyWebContentType('https://example.com/api-reference/endpoints')).toBe(
|
|
1257
|
-
|
|
1258
|
-
|
|
1257
|
+
expect(classifyWebContentType('https://example.com/api-reference/endpoints')).toBe(
|
|
1258
|
+
'documentation-primary'
|
|
1259
|
+
);
|
|
1260
|
+
expect(classifyWebContentType('https://example.com/api-docs/v2')).toBe(
|
|
1261
|
+
'documentation-primary'
|
|
1262
|
+
);
|
|
1263
|
+
expect(classifyWebContentType('https://example.com/apiref/methods')).toBe(
|
|
1264
|
+
'documentation-primary'
|
|
1265
|
+
);
|
|
1259
1266
|
});
|
|
1260
1267
|
|
|
1261
1268
|
it('classifies API reference by title', () => {
|
|
1262
|
-
expect(classifyWebContentType('https://example.com/page', 'API Reference Guide')).toBe(
|
|
1263
|
-
|
|
1269
|
+
expect(classifyWebContentType('https://example.com/page', 'API Reference Guide')).toBe(
|
|
1270
|
+
'documentation-primary'
|
|
1271
|
+
);
|
|
1272
|
+
expect(classifyWebContentType('https://example.com/page', 'Complete API Documentation')).toBe(
|
|
1273
|
+
'documentation-primary'
|
|
1274
|
+
);
|
|
1264
1275
|
});
|
|
1265
1276
|
|
|
1266
1277
|
it('classifies getting started URLs', () => {
|
|
1267
|
-
expect(classifyWebContentType('https://example.com/getting-started')).toBe(
|
|
1268
|
-
|
|
1269
|
-
|
|
1278
|
+
expect(classifyWebContentType('https://example.com/getting-started')).toBe(
|
|
1279
|
+
'documentation-primary'
|
|
1280
|
+
);
|
|
1281
|
+
expect(classifyWebContentType('https://example.com/getting_started')).toBe(
|
|
1282
|
+
'documentation-primary'
|
|
1283
|
+
);
|
|
1284
|
+
expect(classifyWebContentType('https://example.com/gettingstarted')).toBe(
|
|
1285
|
+
'documentation-primary'
|
|
1286
|
+
);
|
|
1270
1287
|
});
|
|
1271
1288
|
|
|
1272
1289
|
it('classifies quickstart URLs', () => {
|
|
1273
|
-
expect(classifyWebContentType('https://example.com/quickstart')).toBe(
|
|
1290
|
+
expect(classifyWebContentType('https://example.com/quickstart')).toBe(
|
|
1291
|
+
'documentation-primary'
|
|
1292
|
+
);
|
|
1274
1293
|
});
|
|
1275
1294
|
|
|
1276
1295
|
it('classifies tutorial URLs', () => {
|
|
1277
|
-
expect(classifyWebContentType('https://example.com/tutorial/basics')).toBe(
|
|
1296
|
+
expect(classifyWebContentType('https://example.com/tutorial/basics')).toBe(
|
|
1297
|
+
'documentation-primary'
|
|
1298
|
+
);
|
|
1278
1299
|
});
|
|
1279
1300
|
|
|
1280
1301
|
it('classifies setup URLs', () => {
|
|
@@ -1282,16 +1303,24 @@ describe('classifyWebContentType', () => {
|
|
|
1282
1303
|
});
|
|
1283
1304
|
|
|
1284
1305
|
it('classifies getting started by title', () => {
|
|
1285
|
-
expect(classifyWebContentType('https://example.com/page', 'Getting Started with React')).toBe(
|
|
1286
|
-
|
|
1287
|
-
|
|
1306
|
+
expect(classifyWebContentType('https://example.com/page', 'Getting Started with React')).toBe(
|
|
1307
|
+
'documentation-primary'
|
|
1308
|
+
);
|
|
1309
|
+
expect(classifyWebContentType('https://example.com/page', 'Quickstart Guide')).toBe(
|
|
1310
|
+
'documentation-primary'
|
|
1311
|
+
);
|
|
1312
|
+
expect(
|
|
1313
|
+
classifyWebContentType('https://example.com/page', 'Tutorial: Build Your First App')
|
|
1314
|
+
).toBe('documentation-primary');
|
|
1288
1315
|
});
|
|
1289
1316
|
});
|
|
1290
1317
|
|
|
1291
1318
|
describe('documentation classification', () => {
|
|
1292
1319
|
it('classifies docs paths', () => {
|
|
1293
1320
|
expect(classifyWebContentType('https://example.com/docs/intro')).toBe('documentation');
|
|
1294
|
-
expect(classifyWebContentType('https://example.com/documentation/advanced')).toBe(
|
|
1321
|
+
expect(classifyWebContentType('https://example.com/documentation/advanced')).toBe(
|
|
1322
|
+
'documentation'
|
|
1323
|
+
);
|
|
1295
1324
|
});
|
|
1296
1325
|
|
|
1297
1326
|
it('classifies reference paths', () => {
|
|
@@ -1307,7 +1336,9 @@ describe('classifyWebContentType', () => {
|
|
|
1307
1336
|
});
|
|
1308
1337
|
|
|
1309
1338
|
it('classifies guide paths', () => {
|
|
1310
|
-
expect(classifyWebContentType('https://example.com/guide/installation')).toBe(
|
|
1339
|
+
expect(classifyWebContentType('https://example.com/guide/installation')).toBe(
|
|
1340
|
+
'documentation'
|
|
1341
|
+
);
|
|
1311
1342
|
});
|
|
1312
1343
|
});
|
|
1313
1344
|
|
|
@@ -1,17 +1,17 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
1
2
|
import { readFile, readdir } from 'node:fs/promises';
|
|
2
3
|
import { join, extname, basename } from 'node:path';
|
|
3
|
-
import {
|
|
4
|
-
import
|
|
5
|
-
import type { EmbeddingEngine } from '../db/embeddings.js';
|
|
6
|
-
import type { Store, FileStore, RepoStore } from '../types/store.js';
|
|
7
|
-
import type { Document } from '../types/document.js';
|
|
4
|
+
import { ChunkingService } from './chunking.service.js';
|
|
5
|
+
import { createLogger } from '../logging/index.js';
|
|
8
6
|
import { createDocumentId } from '../types/brands.js';
|
|
9
|
-
import type { Result } from '../types/result.js';
|
|
10
7
|
import { ok, err } from '../types/result.js';
|
|
11
|
-
import { ChunkingService } from './chunking.service.js';
|
|
12
|
-
import type { ProgressCallback } from '../types/progress.js';
|
|
13
8
|
import type { CodeGraphService } from './code-graph.service.js';
|
|
14
|
-
import {
|
|
9
|
+
import type { EmbeddingEngine } from '../db/embeddings.js';
|
|
10
|
+
import type { LanceStore } from '../db/lance.js';
|
|
11
|
+
import type { Document } from '../types/document.js';
|
|
12
|
+
import type { ProgressCallback } from '../types/progress.js';
|
|
13
|
+
import type { Result } from '../types/result.js';
|
|
14
|
+
import type { Store, FileStore, RepoStore } from '../types/store.js';
|
|
15
15
|
|
|
16
16
|
const logger = createLogger('index-service');
|
|
17
17
|
|
|
@@ -28,9 +28,33 @@ interface IndexOptions {
|
|
|
28
28
|
}
|
|
29
29
|
|
|
30
30
|
const TEXT_EXTENSIONS = new Set([
|
|
31
|
-
'.txt',
|
|
32
|
-
'.
|
|
33
|
-
'.
|
|
31
|
+
'.txt',
|
|
32
|
+
'.md',
|
|
33
|
+
'.js',
|
|
34
|
+
'.ts',
|
|
35
|
+
'.jsx',
|
|
36
|
+
'.tsx',
|
|
37
|
+
'.json',
|
|
38
|
+
'.yaml',
|
|
39
|
+
'.yml',
|
|
40
|
+
'.html',
|
|
41
|
+
'.css',
|
|
42
|
+
'.scss',
|
|
43
|
+
'.less',
|
|
44
|
+
'.py',
|
|
45
|
+
'.rb',
|
|
46
|
+
'.go',
|
|
47
|
+
'.rs',
|
|
48
|
+
'.java',
|
|
49
|
+
'.c',
|
|
50
|
+
'.cpp',
|
|
51
|
+
'.h',
|
|
52
|
+
'.hpp',
|
|
53
|
+
'.sh',
|
|
54
|
+
'.bash',
|
|
55
|
+
'.zsh',
|
|
56
|
+
'.sql',
|
|
57
|
+
'.xml',
|
|
34
58
|
]);
|
|
35
59
|
|
|
36
60
|
export class IndexService {
|
|
@@ -54,39 +78,54 @@ export class IndexService {
|
|
|
54
78
|
}
|
|
55
79
|
|
|
56
80
|
async indexStore(store: Store, onProgress?: ProgressCallback): Promise<Result<IndexResult>> {
|
|
57
|
-
logger.info(
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
81
|
+
logger.info(
|
|
82
|
+
{
|
|
83
|
+
storeId: store.id,
|
|
84
|
+
storeName: store.name,
|
|
85
|
+
storeType: store.type,
|
|
86
|
+
},
|
|
87
|
+
'Starting store indexing'
|
|
88
|
+
);
|
|
62
89
|
|
|
63
90
|
try {
|
|
64
91
|
if (store.type === 'file' || store.type === 'repo') {
|
|
65
92
|
return await this.indexFileStore(store, onProgress);
|
|
66
93
|
}
|
|
67
94
|
|
|
68
|
-
logger.error(
|
|
95
|
+
logger.error(
|
|
96
|
+
{ storeId: store.id, storeType: store.type },
|
|
97
|
+
'Unsupported store type for indexing'
|
|
98
|
+
);
|
|
69
99
|
return err(new Error(`Indexing not supported for store type: ${store.type}`));
|
|
70
100
|
} catch (error) {
|
|
71
|
-
logger.error(
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
101
|
+
logger.error(
|
|
102
|
+
{
|
|
103
|
+
storeId: store.id,
|
|
104
|
+
error: error instanceof Error ? error.message : String(error),
|
|
105
|
+
},
|
|
106
|
+
'Store indexing failed'
|
|
107
|
+
);
|
|
75
108
|
return err(error instanceof Error ? error : new Error(String(error)));
|
|
76
109
|
}
|
|
77
110
|
}
|
|
78
111
|
|
|
79
|
-
private async indexFileStore(
|
|
112
|
+
private async indexFileStore(
|
|
113
|
+
store: FileStore | RepoStore,
|
|
114
|
+
onProgress?: ProgressCallback
|
|
115
|
+
): Promise<Result<IndexResult>> {
|
|
80
116
|
const startTime = Date.now();
|
|
81
117
|
const files = await this.scanDirectory(store.path);
|
|
82
118
|
const documents: Document[] = [];
|
|
83
119
|
let filesProcessed = 0;
|
|
84
120
|
|
|
85
|
-
logger.debug(
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
121
|
+
logger.debug(
|
|
122
|
+
{
|
|
123
|
+
storeId: store.id,
|
|
124
|
+
path: store.path,
|
|
125
|
+
fileCount: files.length,
|
|
126
|
+
},
|
|
127
|
+
'Files scanned for indexing'
|
|
128
|
+
);
|
|
90
129
|
|
|
91
130
|
// Collect source files for code graph building
|
|
92
131
|
const sourceFiles: Array<{ path: string; content: string }> = [];
|
|
@@ -96,7 +135,7 @@ export class IndexService {
|
|
|
96
135
|
type: 'start',
|
|
97
136
|
current: 0,
|
|
98
137
|
total: files.length,
|
|
99
|
-
message: 'Starting index'
|
|
138
|
+
message: 'Starting index',
|
|
100
139
|
});
|
|
101
140
|
|
|
102
141
|
for (const filePath of files) {
|
|
@@ -117,9 +156,10 @@ export class IndexService {
|
|
|
117
156
|
|
|
118
157
|
for (const chunk of chunks) {
|
|
119
158
|
const vector = await this.embeddingEngine.embed(chunk.content);
|
|
120
|
-
const chunkId =
|
|
121
|
-
|
|
122
|
-
|
|
159
|
+
const chunkId =
|
|
160
|
+
chunks.length > 1
|
|
161
|
+
? `${store.id}-${fileHash}-${String(chunk.chunkIndex)}`
|
|
162
|
+
: `${store.id}-${fileHash}`;
|
|
123
163
|
|
|
124
164
|
const doc: Document = {
|
|
125
165
|
id: createDocumentId(chunkId),
|
|
@@ -150,7 +190,7 @@ export class IndexService {
|
|
|
150
190
|
type: 'progress',
|
|
151
191
|
current: filesProcessed,
|
|
152
192
|
total: files.length,
|
|
153
|
-
message: `Indexing ${filePath}
|
|
193
|
+
message: `Indexing ${filePath}`,
|
|
154
194
|
});
|
|
155
195
|
}
|
|
156
196
|
|
|
@@ -169,19 +209,22 @@ export class IndexService {
|
|
|
169
209
|
type: 'complete',
|
|
170
210
|
current: files.length,
|
|
171
211
|
total: files.length,
|
|
172
|
-
message: 'Indexing complete'
|
|
212
|
+
message: 'Indexing complete',
|
|
173
213
|
});
|
|
174
214
|
|
|
175
215
|
const timeMs = Date.now() - startTime;
|
|
176
216
|
|
|
177
|
-
logger.info(
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
217
|
+
logger.info(
|
|
218
|
+
{
|
|
219
|
+
storeId: store.id,
|
|
220
|
+
storeName: store.name,
|
|
221
|
+
documentsIndexed: filesProcessed,
|
|
222
|
+
chunksCreated: documents.length,
|
|
223
|
+
sourceFilesForGraph: sourceFiles.length,
|
|
224
|
+
timeMs,
|
|
225
|
+
},
|
|
226
|
+
'Store indexing complete'
|
|
227
|
+
);
|
|
185
228
|
|
|
186
229
|
return ok({
|
|
187
230
|
documentsIndexed: filesProcessed,
|
|
@@ -287,8 +330,11 @@ export class IndexService {
|
|
|
287
330
|
}
|
|
288
331
|
|
|
289
332
|
// Compiler/transform internals (often not what users want)
|
|
290
|
-
if (
|
|
291
|
-
|
|
333
|
+
if (
|
|
334
|
+
/\/(compiler|transforms?|parse|codegen)\//.test(pathLower) &&
|
|
335
|
+
!fileNameLower.includes('readme') &&
|
|
336
|
+
!fileNameLower.includes('index')
|
|
337
|
+
) {
|
|
292
338
|
return true;
|
|
293
339
|
}
|
|
294
340
|
|
|
@@ -305,14 +351,18 @@ export function classifyWebContentType(url: string, title?: string): string {
|
|
|
305
351
|
const titleLower = (title ?? '').toLowerCase();
|
|
306
352
|
|
|
307
353
|
// API reference documentation → documentation-primary (1.8x boost)
|
|
308
|
-
if (
|
|
309
|
-
|
|
354
|
+
if (
|
|
355
|
+
/\/api[-/]?(ref|reference|docs?)?\//i.test(urlLower) ||
|
|
356
|
+
/api\s*(reference|documentation)/i.test(titleLower)
|
|
357
|
+
) {
|
|
310
358
|
return 'documentation-primary';
|
|
311
359
|
}
|
|
312
360
|
|
|
313
361
|
// Getting started / tutorials → documentation-primary (1.8x boost)
|
|
314
|
-
if (
|
|
315
|
-
|
|
362
|
+
if (
|
|
363
|
+
/\/(getting[-_]?started|quickstart|tutorial|setup)\b/i.test(urlLower) ||
|
|
364
|
+
/(getting started|quickstart|tutorial)/i.test(titleLower)
|
|
365
|
+
) {
|
|
316
366
|
return 'documentation-primary';
|
|
317
367
|
}
|
|
318
368
|
|
package/src/services/index.ts
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
+
import { CodeGraphService } from './code-graph.service.js';
|
|
1
2
|
import { ConfigService } from './config.service.js';
|
|
2
|
-
import { StoreService } from './store.service.js';
|
|
3
|
-
import { SearchService } from './search.service.js';
|
|
4
3
|
import { IndexService } from './index.service.js';
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import { EmbeddingEngine } from '../db/embeddings.js';
|
|
4
|
+
import { SearchService } from './search.service.js';
|
|
5
|
+
import { StoreService } from './store.service.js';
|
|
8
6
|
import { PythonBridge } from '../crawl/bridge.js';
|
|
7
|
+
import { EmbeddingEngine } from '../db/embeddings.js';
|
|
8
|
+
import { LanceStore } from '../db/lance.js';
|
|
9
9
|
import { createLogger, shutdownLogger } from '../logging/index.js';
|
|
10
10
|
|
|
11
11
|
const logger = createLogger('services');
|
|
@@ -42,10 +42,7 @@ export async function createServices(
|
|
|
42
42
|
const resolvedDataDir = config.resolveDataDir();
|
|
43
43
|
|
|
44
44
|
const lance = new LanceStore(resolvedDataDir);
|
|
45
|
-
const embeddings = new EmbeddingEngine(
|
|
46
|
-
appConfig.embedding.model,
|
|
47
|
-
appConfig.embedding.dimensions
|
|
48
|
-
);
|
|
45
|
+
const embeddings = new EmbeddingEngine(appConfig.embedding.model, appConfig.embedding.dimensions);
|
|
49
46
|
|
|
50
47
|
await embeddings.initialize();
|
|
51
48
|
|
|
@@ -30,7 +30,7 @@ describe('JobService', () => {
|
|
|
30
30
|
it('should create a job with required fields', () => {
|
|
31
31
|
const job = jobService.createJob({
|
|
32
32
|
type: 'clone',
|
|
33
|
-
details: { storeId: 'test-store', url: 'https://github.com/test/repo' }
|
|
33
|
+
details: { storeId: 'test-store', url: 'https://github.com/test/repo' },
|
|
34
34
|
});
|
|
35
35
|
|
|
36
36
|
expect(job.id).toMatch(/^job_[a-f0-9]{12}$/);
|
|
@@ -45,7 +45,7 @@ describe('JobService', () => {
|
|
|
45
45
|
it('should persist job to file', () => {
|
|
46
46
|
const job = jobService.createJob({
|
|
47
47
|
type: 'index',
|
|
48
|
-
details: { storeId: 'test' }
|
|
48
|
+
details: { storeId: 'test' },
|
|
49
49
|
});
|
|
50
50
|
|
|
51
51
|
const jobFile = join(tempDir, 'jobs', `${job.id}.json`);
|
|
@@ -57,13 +57,13 @@ describe('JobService', () => {
|
|
|
57
57
|
it('should update job status and progress', () => {
|
|
58
58
|
const job = jobService.createJob({
|
|
59
59
|
type: 'index',
|
|
60
|
-
details: { storeId: 'test' }
|
|
60
|
+
details: { storeId: 'test' },
|
|
61
61
|
});
|
|
62
62
|
|
|
63
63
|
jobService.updateJob(job.id, {
|
|
64
64
|
status: 'running',
|
|
65
65
|
progress: 50,
|
|
66
|
-
message: 'Processing files...'
|
|
66
|
+
message: 'Processing files...',
|
|
67
67
|
});
|
|
68
68
|
|
|
69
69
|
const updated = jobService.getJob(job.id);
|
|
@@ -75,11 +75,11 @@ describe('JobService', () => {
|
|
|
75
75
|
it('should merge job details', () => {
|
|
76
76
|
const job = jobService.createJob({
|
|
77
77
|
type: 'index',
|
|
78
|
-
details: { storeId: 'test', filesProcessed: 10 }
|
|
78
|
+
details: { storeId: 'test', filesProcessed: 10 },
|
|
79
79
|
});
|
|
80
80
|
|
|
81
81
|
jobService.updateJob(job.id, {
|
|
82
|
-
details: { totalFiles: 100 }
|
|
82
|
+
details: { totalFiles: 100 },
|
|
83
83
|
});
|
|
84
84
|
|
|
85
85
|
const updated = jobService.getJob(job.id);
|
|
@@ -99,7 +99,7 @@ describe('JobService', () => {
|
|
|
99
99
|
it('should retrieve job by ID', () => {
|
|
100
100
|
const job = jobService.createJob({
|
|
101
101
|
type: 'clone',
|
|
102
|
-
details: { storeId: 'test' }
|
|
102
|
+
details: { storeId: 'test' },
|
|
103
103
|
});
|
|
104
104
|
|
|
105
105
|
const retrieved = jobService.getJob(job.id);
|
|
@@ -193,7 +193,7 @@ describe('JobService', () => {
|
|
|
193
193
|
it('should cancel a pending job', () => {
|
|
194
194
|
const job = jobService.createJob({
|
|
195
195
|
type: 'index',
|
|
196
|
-
details: { storeId: 'test' }
|
|
196
|
+
details: { storeId: 'test' },
|
|
197
197
|
});
|
|
198
198
|
|
|
199
199
|
const result = jobService.cancelJob(job.id);
|
|
@@ -211,7 +211,7 @@ describe('JobService', () => {
|
|
|
211
211
|
it('should return error for completed job', () => {
|
|
212
212
|
const job = jobService.createJob({
|
|
213
213
|
type: 'index',
|
|
214
|
-
details: { storeId: 'test' }
|
|
214
|
+
details: { storeId: 'test' },
|
|
215
215
|
});
|
|
216
216
|
jobService.updateJob(job.id, { status: 'completed' });
|
|
217
217
|
|
|
@@ -222,7 +222,7 @@ describe('JobService', () => {
|
|
|
222
222
|
it('should return error for failed job', () => {
|
|
223
223
|
const job = jobService.createJob({
|
|
224
224
|
type: 'index',
|
|
225
|
-
details: { storeId: 'test' }
|
|
225
|
+
details: { storeId: 'test' },
|
|
226
226
|
});
|
|
227
227
|
jobService.updateJob(job.id, { status: 'failed' });
|
|
228
228
|
|
|
@@ -233,7 +233,7 @@ describe('JobService', () => {
|
|
|
233
233
|
it('should succeed if job already cancelled', () => {
|
|
234
234
|
const job = jobService.createJob({
|
|
235
235
|
type: 'index',
|
|
236
|
-
details: { storeId: 'test' }
|
|
236
|
+
details: { storeId: 'test' },
|
|
237
237
|
});
|
|
238
238
|
jobService.cancelJob(job.id);
|
|
239
239
|
|
|
@@ -244,7 +244,7 @@ describe('JobService', () => {
|
|
|
244
244
|
it('should remove PID file if exists', () => {
|
|
245
245
|
const job = jobService.createJob({
|
|
246
246
|
type: 'index',
|
|
247
|
-
details: { storeId: 'test' }
|
|
247
|
+
details: { storeId: 'test' },
|
|
248
248
|
});
|
|
249
249
|
|
|
250
250
|
// Create a PID file with a non-existent process ID (NOT our own PID - that would kill the test runner!)
|
|
@@ -259,7 +259,7 @@ describe('JobService', () => {
|
|
|
259
259
|
it('should handle missing PID file gracefully', () => {
|
|
260
260
|
const job = jobService.createJob({
|
|
261
261
|
type: 'index',
|
|
262
|
-
details: { storeId: 'test' }
|
|
262
|
+
details: { storeId: 'test' },
|
|
263
263
|
});
|
|
264
264
|
|
|
265
265
|
const result = jobService.cancelJob(job.id);
|
|
@@ -269,7 +269,7 @@ describe('JobService', () => {
|
|
|
269
269
|
it('should handle invalid PID gracefully', () => {
|
|
270
270
|
const job = jobService.createJob({
|
|
271
271
|
type: 'index',
|
|
272
|
-
details: { storeId: 'test' }
|
|
272
|
+
details: { storeId: 'test' },
|
|
273
273
|
});
|
|
274
274
|
|
|
275
275
|
// Create PID file with non-existent process ID
|
|
@@ -285,7 +285,7 @@ describe('JobService', () => {
|
|
|
285
285
|
it('should clean up old completed jobs', () => {
|
|
286
286
|
const job = jobService.createJob({
|
|
287
287
|
type: 'index',
|
|
288
|
-
details: { storeId: 'test' }
|
|
288
|
+
details: { storeId: 'test' },
|
|
289
289
|
});
|
|
290
290
|
jobService.updateJob(job.id, { status: 'completed' });
|
|
291
291
|
|
|
@@ -303,7 +303,7 @@ describe('JobService', () => {
|
|
|
303
303
|
it('should clean up old failed jobs', () => {
|
|
304
304
|
const job = jobService.createJob({
|
|
305
305
|
type: 'index',
|
|
306
|
-
details: { storeId: 'test' }
|
|
306
|
+
details: { storeId: 'test' },
|
|
307
307
|
});
|
|
308
308
|
jobService.updateJob(job.id, { status: 'failed' });
|
|
309
309
|
|
|
@@ -319,7 +319,7 @@ describe('JobService', () => {
|
|
|
319
319
|
it('should clean up old cancelled jobs', () => {
|
|
320
320
|
const job = jobService.createJob({
|
|
321
321
|
type: 'index',
|
|
322
|
-
details: { storeId: 'test' }
|
|
322
|
+
details: { storeId: 'test' },
|
|
323
323
|
});
|
|
324
324
|
jobService.updateJob(job.id, { status: 'cancelled' });
|
|
325
325
|
|
|
@@ -335,7 +335,7 @@ describe('JobService', () => {
|
|
|
335
335
|
it('should not clean up recent jobs', () => {
|
|
336
336
|
const job = jobService.createJob({
|
|
337
337
|
type: 'index',
|
|
338
|
-
details: { storeId: 'test' }
|
|
338
|
+
details: { storeId: 'test' },
|
|
339
339
|
});
|
|
340
340
|
jobService.updateJob(job.id, { status: 'completed' });
|
|
341
341
|
|
|
@@ -346,7 +346,7 @@ describe('JobService', () => {
|
|
|
346
346
|
it('should not clean up active jobs', () => {
|
|
347
347
|
const job = jobService.createJob({
|
|
348
348
|
type: 'index',
|
|
349
|
-
details: { storeId: 'test' }
|
|
349
|
+
details: { storeId: 'test' },
|
|
350
350
|
});
|
|
351
351
|
jobService.updateJob(job.id, { status: 'running' });
|
|
352
352
|
|
|
@@ -363,7 +363,7 @@ describe('JobService', () => {
|
|
|
363
363
|
it('should use default 24 hours if not specified', () => {
|
|
364
364
|
const job = jobService.createJob({
|
|
365
365
|
type: 'index',
|
|
366
|
-
details: { storeId: 'test' }
|
|
366
|
+
details: { storeId: 'test' },
|
|
367
367
|
});
|
|
368
368
|
jobService.updateJob(job.id, { status: 'completed' });
|
|
369
369
|
|
|
@@ -384,7 +384,7 @@ describe('JobService', () => {
|
|
|
384
384
|
jobService.updateJob(job2.id, { status: 'failed' });
|
|
385
385
|
|
|
386
386
|
// Make both old
|
|
387
|
-
[job1, job2].forEach(job => {
|
|
387
|
+
[job1, job2].forEach((job) => {
|
|
388
388
|
const jobFile = join(tempDir, 'jobs', `${job.id}.json`);
|
|
389
389
|
const jobData = JSON.parse(readFileSync(jobFile, 'utf-8'));
|
|
390
390
|
jobData.updatedAt = new Date(Date.now() - 48 * 60 * 60 * 1000).toISOString();
|
|
@@ -400,7 +400,7 @@ describe('JobService', () => {
|
|
|
400
400
|
it('should delete a job file', () => {
|
|
401
401
|
const job = jobService.createJob({
|
|
402
402
|
type: 'index',
|
|
403
|
-
details: { storeId: 'test' }
|
|
403
|
+
details: { storeId: 'test' },
|
|
404
404
|
});
|
|
405
405
|
|
|
406
406
|
const deleted = jobService.deleteJob(job.id);
|
|
@@ -1,23 +1,20 @@
|
|
|
1
|
+
import { randomUUID } from 'crypto';
|
|
1
2
|
import fs from 'fs';
|
|
2
3
|
import path from 'path';
|
|
3
|
-
import { randomUUID } from 'crypto';
|
|
4
|
-
import type {
|
|
5
|
-
Job,
|
|
6
|
-
CreateJobParams,
|
|
7
|
-
UpdateJobParams,
|
|
8
|
-
JobStatus
|
|
9
|
-
} from '../types/job.js';
|
|
10
4
|
import { Result, ok, err } from '../types/result.js';
|
|
5
|
+
import type { Job, CreateJobParams, UpdateJobParams, JobStatus } from '../types/job.js';
|
|
11
6
|
|
|
12
7
|
export class JobService {
|
|
13
8
|
private readonly jobsDir: string;
|
|
14
9
|
|
|
15
10
|
constructor(dataDir?: string) {
|
|
16
11
|
// Default to ~/.local/share/bluera-knowledge/jobs
|
|
17
|
-
const baseDir =
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
12
|
+
const baseDir =
|
|
13
|
+
dataDir ??
|
|
14
|
+
path.join(
|
|
15
|
+
process.env['HOME'] ?? process.env['USERPROFILE'] ?? '.',
|
|
16
|
+
'.local/share/bluera-knowledge'
|
|
17
|
+
);
|
|
21
18
|
this.jobsDir = path.join(baseDir, 'jobs');
|
|
22
19
|
|
|
23
20
|
// Ensure jobs directory exists
|
|
@@ -38,7 +35,7 @@ export class JobService {
|
|
|
38
35
|
message: params.message ?? `${params.type} job created`,
|
|
39
36
|
details: params.details,
|
|
40
37
|
createdAt: new Date().toISOString(),
|
|
41
|
-
updatedAt: new Date().toISOString()
|
|
38
|
+
updatedAt: new Date().toISOString(),
|
|
42
39
|
};
|
|
43
40
|
|
|
44
41
|
// Write job to file
|
|
@@ -132,9 +129,7 @@ export class JobService {
|
|
|
132
129
|
}
|
|
133
130
|
|
|
134
131
|
// Sort by updated time (most recent first)
|
|
135
|
-
jobs.sort((a, b) =>
|
|
136
|
-
new Date(b.updatedAt).getTime() - new Date(a.updatedAt).getTime()
|
|
137
|
-
);
|
|
132
|
+
jobs.sort((a, b) => new Date(b.updatedAt).getTime() - new Date(a.updatedAt).getTime());
|
|
138
133
|
|
|
139
134
|
return jobs;
|
|
140
135
|
}
|
|
@@ -168,7 +163,7 @@ export class JobService {
|
|
|
168
163
|
this.updateJob(jobId, {
|
|
169
164
|
status: 'cancelled',
|
|
170
165
|
message: 'Job cancelled by user',
|
|
171
|
-
details: { cancelledAt: new Date().toISOString() }
|
|
166
|
+
details: { cancelledAt: new Date().toISOString() },
|
|
172
167
|
});
|
|
173
168
|
|
|
174
169
|
// Kill worker process if it exists
|
|
@@ -176,7 +171,12 @@ export class JobService {
|
|
|
176
171
|
if (fs.existsSync(pidFile)) {
|
|
177
172
|
try {
|
|
178
173
|
const pid = parseInt(fs.readFileSync(pidFile, 'utf-8'), 10);
|
|
179
|
-
|
|
174
|
+
// Validate PID: must be positive integer > 0
|
|
175
|
+
// PID 0 = sends to process group (DANGEROUS - kills terminal!)
|
|
176
|
+
// Negative PIDs have special meanings in kill()
|
|
177
|
+
if (!Number.isNaN(pid) && Number.isInteger(pid) && pid > 0) {
|
|
178
|
+
process.kill(pid, 'SIGTERM');
|
|
179
|
+
}
|
|
180
180
|
} catch {
|
|
181
181
|
// Process may have already exited, ignore
|
|
182
182
|
}
|
|
@@ -196,7 +196,7 @@ export class JobService {
|
|
|
196
196
|
*/
|
|
197
197
|
cleanupOldJobs(olderThanHours: number = 24): number {
|
|
198
198
|
const jobs = this.listJobs();
|
|
199
|
-
const cutoffTime = Date.now() -
|
|
199
|
+
const cutoffTime = Date.now() - olderThanHours * 60 * 60 * 1000;
|
|
200
200
|
let cleaned = 0;
|
|
201
201
|
|
|
202
202
|
for (const job of jobs) {
|
|
@@ -439,9 +439,7 @@ describe('ProjectRootService', () => {
|
|
|
439
439
|
it('handles concurrent resolve calls', () => {
|
|
440
440
|
process.env.PROJECT_ROOT = '/test/path';
|
|
441
441
|
|
|
442
|
-
const results = Array.from({ length: 10 }, () =>
|
|
443
|
-
ProjectRootService.resolve()
|
|
444
|
-
);
|
|
442
|
+
const results = Array.from({ length: 10 }, () => ProjectRootService.resolve());
|
|
445
443
|
|
|
446
444
|
results.forEach((result) => {
|
|
447
445
|
expect(result).toBe(path.normalize('/test/path'));
|