docorbit 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +660 -0
- package/apps/cli/bin/docorbit.js +8 -0
- package/apps/cli/src/commands/add.ts +44 -0
- package/apps/cli/src/commands/api.ts +38 -0
- package/apps/cli/src/commands/context.ts +47 -0
- package/apps/cli/src/commands/dashboard.ts +55 -0
- package/apps/cli/src/commands/diff.ts +30 -0
- package/apps/cli/src/commands/evaluate.ts +133 -0
- package/apps/cli/src/commands/examples.ts +39 -0
- package/apps/cli/src/commands/export.ts +89 -0
- package/apps/cli/src/commands/impact.ts +31 -0
- package/apps/cli/src/commands/init.ts +69 -0
- package/apps/cli/src/commands/inspect.ts +30 -0
- package/apps/cli/src/commands/mcp.ts +72 -0
- package/apps/cli/src/commands/pitfalls.ts +38 -0
- package/apps/cli/src/commands/recipes.ts +35 -0
- package/apps/cli/src/commands/search.ts +48 -0
- package/apps/cli/src/commands/update.ts +73 -0
- package/apps/cli/src/commands/verify.ts +48 -0
- package/apps/cli/src/formatters/colors.ts +23 -0
- package/apps/cli/src/formatters/inspection.ts +102 -0
- package/apps/cli/src/formatters/knowledge.ts +272 -0
- package/apps/cli/src/formatters/retrieval.ts +74 -0
- package/apps/cli/src/formatters/terminal.ts +6 -0
- package/apps/cli/src/formatters/verification.ts +126 -0
- package/apps/cli/src/index.ts +409 -0
- package/bin/docorbit.js +8 -0
- package/package.json +46 -0
- package/packages/core/src/dashboard/server.ts +314 -0
- package/packages/core/src/dashboard/ui.ts +586 -0
- package/packages/core/src/implementation-service.ts +451 -0
- package/packages/core/src/index.ts +7 -0
- package/packages/core/src/inspector.ts +71 -0
- package/packages/core/src/pipeline.ts +331 -0
- package/packages/crawler/src/config.ts +12 -0
- package/packages/crawler/src/fetcher.ts +185 -0
- package/packages/crawler/src/index.ts +2 -0
- package/packages/discovery/src/index.ts +31 -0
- package/packages/discovery/src/provider.ts +47 -0
- package/packages/discovery/src/providers/generic.ts +98 -0
- package/packages/discovery/src/providers/github.ts +61 -0
- package/packages/discovery/src/providers/llms-txt.ts +73 -0
- package/packages/discovery/src/providers/markdown.ts +48 -0
- package/packages/discovery/src/providers/openapi.ts +91 -0
- package/packages/discovery/src/providers/sitemap.ts +62 -0
- package/packages/discovery/src/providers/skill.ts +54 -0
- package/packages/discovery/src/ranker.ts +123 -0
- package/packages/evaluation/src/dataset.ts +963 -0
- package/packages/evaluation/src/index.ts +8 -0
- package/packages/evaluation/src/runner.ts +241 -0
- package/packages/evaluation/src/strategies/context7-runner.ts +269 -0
- package/packages/evaluation/src/strategies/docorbit-runner.ts +228 -0
- package/packages/evaluation/src/strategies/firecrawl-runner.ts +172 -0
- package/packages/evaluation/src/strategies/web-search-runner.ts +194 -0
- package/packages/evaluation/src/types.ts +34 -0
- package/packages/evaluation/src/version-matcher.ts +73 -0
- package/packages/export/src/agents-md.ts +200 -0
- package/packages/export/src/claude-md.ts +141 -0
- package/packages/export/src/docs-map.ts +150 -0
- package/packages/export/src/index.ts +6 -0
- package/packages/export/src/llms-txt.ts +96 -0
- package/packages/export/src/service.ts +250 -0
- package/packages/export/src/skill-md.ts +128 -0
- package/packages/mcp/src/index.ts +46 -0
- package/packages/mcp/src/resources/index.ts +189 -0
- package/packages/mcp/src/server.ts +278 -0
- package/packages/mcp/src/tools/analyze-impact.ts +74 -0
- package/packages/mcp/src/tools/check-api.ts +86 -0
- package/packages/mcp/src/tools/diff-docs.ts +68 -0
- package/packages/mcp/src/tools/export-context.ts +73 -0
- package/packages/mcp/src/tools/find-api.ts +99 -0
- package/packages/mcp/src/tools/find-example.ts +100 -0
- package/packages/mcp/src/tools/find-pitfall.ts +94 -0
- package/packages/mcp/src/tools/find-recipe.ts +98 -0
- package/packages/mcp/src/tools/get-doc.ts +130 -0
- package/packages/mcp/src/tools/get-docs-map.ts +64 -0
- package/packages/mcp/src/tools/get-version.ts +118 -0
- package/packages/mcp/src/tools/implementation-context.ts +88 -0
- package/packages/mcp/src/tools/index.ts +59 -0
- package/packages/mcp/src/tools/list-sources.ts +85 -0
- package/packages/mcp/src/tools/search-docs.ts +123 -0
- package/packages/mcp/src/tools/types.ts +28 -0
- package/packages/mcp/src/transports/http.ts +256 -0
- package/packages/mcp/src/transports/stdio.ts +105 -0
- package/packages/mcp/src/transports/types.ts +6 -0
- package/packages/mcp/src/types.ts +102 -0
- package/packages/normalizer/src/example-indexer.ts +240 -0
- package/packages/normalizer/src/html.ts +253 -0
- package/packages/normalizer/src/index.ts +8 -0
- package/packages/normalizer/src/llms.ts +83 -0
- package/packages/normalizer/src/openapi/endpoint-parser.ts +406 -0
- package/packages/normalizer/src/openapi/schema-resolver.ts +111 -0
- package/packages/normalizer/src/openapi.ts +2 -0
- package/packages/normalizer/src/page.ts +184 -0
- package/packages/normalizer/src/pitfall-extractor.ts +190 -0
- package/packages/normalizer/src/slicer.ts +455 -0
- package/packages/retrieval/src/engine.ts +120 -0
- package/packages/retrieval/src/index.ts +7 -0
- package/packages/retrieval/src/intent.ts +43 -0
- package/packages/retrieval/src/packer.ts +145 -0
- package/packages/retrieval/src/recipe-engine.ts +313 -0
- package/packages/retrieval/src/scorer.ts +139 -0
- package/packages/retrieval/src/weights.ts +31 -0
- package/packages/security/src/annotations.ts +112 -0
- package/packages/security/src/index.ts +2 -0
- package/packages/security/src/ssrf.ts +153 -0
- package/packages/shared/src/errors.ts +53 -0
- package/packages/shared/src/hashing.ts +23 -0
- package/packages/shared/src/index.ts +3 -0
- package/packages/shared/src/types.ts +881 -0
- package/packages/storage/src/db.ts +72 -0
- package/packages/storage/src/index.ts +11 -0
- package/packages/storage/src/interfaces.ts +115 -0
- package/packages/storage/src/repositories/api-repository.ts +219 -0
- package/packages/storage/src/repositories/chunk-repository.ts +316 -0
- package/packages/storage/src/repositories/example-repository.ts +206 -0
- package/packages/storage/src/repositories/page-repository.ts +205 -0
- package/packages/storage/src/repositories/pitfall-repository.ts +188 -0
- package/packages/storage/src/repositories/source-repository.ts +205 -0
- package/packages/storage/src/repository.ts +256 -0
- package/packages/storage/src/schema.ts +269 -0
- package/packages/storage/src/search-tokens.ts +28 -0
- package/packages/verification/src/diff-engine.ts +258 -0
- package/packages/verification/src/extractor.ts +339 -0
- package/packages/verification/src/impact-scanner.ts +203 -0
- package/packages/verification/src/index.ts +5 -0
- package/packages/verification/src/services.ts +238 -0
- package/packages/verification/src/verifier.ts +375 -0
- package/packages/workspace/src/detector.ts +143 -0
- package/packages/workspace/src/ecosystems/cargo.ts +84 -0
- package/packages/workspace/src/ecosystems/composer.ts +42 -0
- package/packages/workspace/src/ecosystems/go.ts +54 -0
- package/packages/workspace/src/ecosystems/index.ts +34 -0
- package/packages/workspace/src/ecosystems/maven.ts +34 -0
- package/packages/workspace/src/ecosystems/npm.ts +83 -0
- package/packages/workspace/src/ecosystems/pub.ts +40 -0
- package/packages/workspace/src/ecosystems/pypi.ts +100 -0
- package/packages/workspace/src/ecosystems/rubygems.ts +30 -0
- package/packages/workspace/src/ecosystems/types.ts +18 -0
- package/packages/workspace/src/index.ts +5 -0
- package/packages/workspace/src/lockfile.ts +194 -0
- package/packages/workspace/src/resolver.ts +234 -0
- package/packages/workspace/src/semver.ts +259 -0
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { DocOrbitDb, DocOrbitRepository } from '../../../../packages/storage/src/index.ts';
|
|
2
|
+
import { IngestionPipeline } from '../../../../packages/core/src/index.ts';
|
|
3
|
+
import { formatIngestionResult } from '../formatters/terminal.ts';
|
|
4
|
+
|
|
5
|
+
export interface AddCommandOptions {
|
|
6
|
+
json?: boolean;
|
|
7
|
+
dbPath?: string;
|
|
8
|
+
maxPages?: number;
|
|
9
|
+
allowLocalhost?: boolean;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export async function runAddCommand(targetUrl: string, options: AddCommandOptions = {}): Promise<void> {
|
|
13
|
+
if (!targetUrl) {
|
|
14
|
+
console.error('Error: Please provide a documentation target URL.');
|
|
15
|
+
console.error('Usage: docorbit add <url> [--json] [--db <path>]');
|
|
16
|
+
process.exit(1);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
const dbPath = options.dbPath || '.docorbit/docorbit.db';
|
|
20
|
+
const db = new DocOrbitDb(dbPath);
|
|
21
|
+
const repository = new DocOrbitRepository(db);
|
|
22
|
+
|
|
23
|
+
try {
|
|
24
|
+
const pipeline = new IngestionPipeline(repository, {
|
|
25
|
+
allowLocalhostForTesting: options.allowLocalhost,
|
|
26
|
+
crawlerConfig: {
|
|
27
|
+
maxPages: options.maxPages || 50,
|
|
28
|
+
},
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
const result = await pipeline.ingest(targetUrl);
|
|
32
|
+
|
|
33
|
+
if (options.json) {
|
|
34
|
+
console.log(JSON.stringify(result, null, 2));
|
|
35
|
+
} else {
|
|
36
|
+
console.log(formatIngestionResult(result));
|
|
37
|
+
}
|
|
38
|
+
} catch (err: unknown) {
|
|
39
|
+
console.error(`DocOrbit Ingestion Error: ${err instanceof Error ? err.message : String(err)}`);
|
|
40
|
+
process.exit(1);
|
|
41
|
+
} finally {
|
|
42
|
+
db.close();
|
|
43
|
+
}
|
|
44
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { DocOrbitDb, DocOrbitRepository } from '../../../../packages/storage/src/index.ts';
|
|
2
|
+
import { resolveProjectContext } from '../../../../packages/workspace/src/index.ts';
|
|
3
|
+
import type { ApiCommandOptions } from '../../../../packages/shared/src/index.ts';
|
|
4
|
+
import { formatApiEndpoints } from '../formatters/terminal.ts';
|
|
5
|
+
|
|
6
|
+
export async function runApiCommand(query: string = '', options: ApiCommandOptions = {}): Promise<void> {
|
|
7
|
+
const dbPath = options.dbPath || '.docorbit/docorbit.db';
|
|
8
|
+
const db = new DocOrbitDb(dbPath);
|
|
9
|
+
const repository = new DocOrbitRepository(db);
|
|
10
|
+
|
|
11
|
+
try {
|
|
12
|
+
let effectiveDocVersion = options.docVersion;
|
|
13
|
+
|
|
14
|
+
if (options.projectDir) {
|
|
15
|
+
const projContext = resolveProjectContext(options.projectDir, query, repository);
|
|
16
|
+
if (projContext.matchedDependency && !effectiveDocVersion) {
|
|
17
|
+
effectiveDocVersion = projContext.matchedDependency.targetDocVersion;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const endpoints = repository.searchApiEndpoints(query, {
|
|
22
|
+
method: options.method,
|
|
23
|
+
docVersion: effectiveDocVersion,
|
|
24
|
+
limit: options.limit || 20,
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
if (options.json) {
|
|
28
|
+
console.log(JSON.stringify(endpoints, null, 2));
|
|
29
|
+
} else {
|
|
30
|
+
console.log(formatApiEndpoints(endpoints, query));
|
|
31
|
+
}
|
|
32
|
+
} catch (err: unknown) {
|
|
33
|
+
console.error(`DocOrbit API Error: ${err instanceof Error ? err.message : String(err)}`);
|
|
34
|
+
process.exit(1);
|
|
35
|
+
} finally {
|
|
36
|
+
db.close();
|
|
37
|
+
}
|
|
38
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { DocOrbitDb, DocOrbitRepository } from '../../../../packages/storage/src/index.ts';
|
|
2
|
+
import { RetrievalEngine } from '../../../../packages/retrieval/src/index.ts';
|
|
3
|
+
import { formatContextPackage } from '../formatters/terminal.ts';
|
|
4
|
+
|
|
5
|
+
export interface ContextCommandOptions {
|
|
6
|
+
json?: boolean;
|
|
7
|
+
dbPath?: string;
|
|
8
|
+
tokens?: number;
|
|
9
|
+
maxChunks?: number;
|
|
10
|
+
snapshotId?: string;
|
|
11
|
+
docVersion?: string;
|
|
12
|
+
projectDir?: string;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export async function runContextCommand(task: string, options: ContextCommandOptions = {}): Promise<void> {
|
|
16
|
+
if (!task) {
|
|
17
|
+
console.error('Error: Please provide a coding task description.');
|
|
18
|
+
console.error('Usage: docorbit context "<task>" [--tokens <n>] [--doc-version <ver>] [--project <dir>] [--json] [--db <path>]');
|
|
19
|
+
process.exit(1);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const dbPath = options.dbPath || '.docorbit/docorbit.db';
|
|
23
|
+
const db = new DocOrbitDb(dbPath);
|
|
24
|
+
const repository = new DocOrbitRepository(db);
|
|
25
|
+
const engine = new RetrievalEngine(repository);
|
|
26
|
+
|
|
27
|
+
try {
|
|
28
|
+
const pkg = await engine.buildContext(task, {
|
|
29
|
+
tokenBudget: options.tokens || 3000,
|
|
30
|
+
maxChunks: options.maxChunks,
|
|
31
|
+
snapshotId: options.snapshotId,
|
|
32
|
+
docVersion: options.docVersion,
|
|
33
|
+
projectDir: options.projectDir,
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
if (options.json) {
|
|
37
|
+
console.log(JSON.stringify(pkg, null, 2));
|
|
38
|
+
} else {
|
|
39
|
+
console.log(formatContextPackage(pkg));
|
|
40
|
+
}
|
|
41
|
+
} catch (err: unknown) {
|
|
42
|
+
console.error(`DocOrbit Context Error: ${err instanceof Error ? err.message : String(err)}`);
|
|
43
|
+
process.exit(1);
|
|
44
|
+
} finally {
|
|
45
|
+
db.close();
|
|
46
|
+
}
|
|
47
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import { DocOrbitDb, DocOrbitRepository } from '../../../../packages/storage/src/index.ts';
|
|
3
|
+
import { DashboardServer } from '../../../../packages/core/src/index.ts';
|
|
4
|
+
import { c } from '../formatters/colors.ts';
|
|
5
|
+
|
|
6
|
+
export interface DashboardCommandOptions {
|
|
7
|
+
port?: number;
|
|
8
|
+
host?: string;
|
|
9
|
+
projectDir?: string;
|
|
10
|
+
dbPath?: string;
|
|
11
|
+
noOpen?: boolean;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export async function runDashboardCommand(options: DashboardCommandOptions = {}): Promise<void> {
|
|
15
|
+
const projectDir = options.projectDir ? path.resolve(options.projectDir) : process.cwd();
|
|
16
|
+
const dbPath = options.dbPath || path.join(projectDir, '.docorbit', 'docorbit.db');
|
|
17
|
+
const port = options.port || 3737;
|
|
18
|
+
const host = options.host || '127.0.0.1';
|
|
19
|
+
|
|
20
|
+
const db = new DocOrbitDb(dbPath);
|
|
21
|
+
const repo = new DocOrbitRepository(db);
|
|
22
|
+
|
|
23
|
+
const server = new DashboardServer({
|
|
24
|
+
repo,
|
|
25
|
+
projectDir,
|
|
26
|
+
dbPath,
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
try {
|
|
30
|
+
const info = await server.start(port, host);
|
|
31
|
+
console.log(`\n${c.bold(c.cyan('DocOrbit Local Dashboard'))}`);
|
|
32
|
+
console.log(`${c.dim('Documentation Intelligence & Verification Layer')}\n`);
|
|
33
|
+
console.log(` ${c.green('●')} Server running at: ${c.bold(c.underline(info.url))}`);
|
|
34
|
+
console.log(` ${c.dim('Database:')} ${dbPath}`);
|
|
35
|
+
console.log(` ${c.dim('Workspace:')} ${projectDir}`);
|
|
36
|
+
console.log(` ${c.dim('Security:')} ${c.yellow('untrusted documentation boundary enforced')}\n`);
|
|
37
|
+
console.log(c.dim('Press Ctrl+C to stop.\n'));
|
|
38
|
+
|
|
39
|
+
// Keep process alive until interrupted
|
|
40
|
+
await new Promise<void>((resolve) => {
|
|
41
|
+
const shutdown = async () => {
|
|
42
|
+
console.log(`\n${c.dim('Stopping dashboard server...')}`);
|
|
43
|
+
await server.stop();
|
|
44
|
+
db.close();
|
|
45
|
+
resolve();
|
|
46
|
+
};
|
|
47
|
+
process.on('SIGINT', shutdown);
|
|
48
|
+
process.on('SIGTERM', shutdown);
|
|
49
|
+
});
|
|
50
|
+
} catch (err: any) {
|
|
51
|
+
db.close();
|
|
52
|
+
console.error(c.red(`Failed to start dashboard server: ${err.message}`));
|
|
53
|
+
process.exit(1);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { DocOrbitDb, DocOrbitRepository } from '../../../../packages/storage/src/index.ts';
|
|
2
|
+
import { DiffService } from '../../../../packages/verification/src/index.ts';
|
|
3
|
+
import type { DiffCommandOptions } from '../../../../packages/shared/src/index.ts';
|
|
4
|
+
import { formatDiffReport } from '../formatters/terminal.ts';
|
|
5
|
+
|
|
6
|
+
export async function runDiffCommand(target: string = '', options: DiffCommandOptions = {}): Promise<void> {
|
|
7
|
+
const dbPath = options.dbPath || '.docorbit/docorbit.db';
|
|
8
|
+
const db = new DocOrbitDb(dbPath);
|
|
9
|
+
const repo = new DocOrbitRepository(db);
|
|
10
|
+
|
|
11
|
+
try {
|
|
12
|
+
const service = new DiffService(repo);
|
|
13
|
+
const { result } = service.diffDocs({
|
|
14
|
+
fromVersion: options.from,
|
|
15
|
+
toVersion: options.to,
|
|
16
|
+
sourceId: options.source || (target && !target.startsWith('-') ? target : undefined),
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
if (options.json) {
|
|
20
|
+
console.log(JSON.stringify(result, null, 2));
|
|
21
|
+
} else {
|
|
22
|
+
console.log(formatDiffReport(result));
|
|
23
|
+
}
|
|
24
|
+
} catch (err: unknown) {
|
|
25
|
+
console.error(`DocOrbit Diff Error: ${err instanceof Error ? err.message : String(err)}`);
|
|
26
|
+
process.exit(1);
|
|
27
|
+
} finally {
|
|
28
|
+
db.close();
|
|
29
|
+
}
|
|
30
|
+
}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { BenchmarkRunner } from '../../../../packages/evaluation/src/index.ts';
|
|
2
|
+
import type { AgentEvaluationStrategy, EvaluationSplit } from '../../../../packages/evaluation/src/index.ts';
|
|
3
|
+
import { c } from '../formatters/colors.ts';
|
|
4
|
+
|
|
5
|
+
export interface EvaluateCommandOptions {
|
|
6
|
+
split?: string;
|
|
7
|
+
strategy?: string;
|
|
8
|
+
task?: string;
|
|
9
|
+
output?: string;
|
|
10
|
+
json?: boolean;
|
|
11
|
+
verbose?: boolean;
|
|
12
|
+
/** Offline simulation mode — no network required, for CI regression only */
|
|
13
|
+
simulation?: boolean;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export async function handleEvaluateCommand(options: EvaluateCommandOptions = {}): Promise<void> {
|
|
17
|
+
const sim = options.simulation ?? false;
|
|
18
|
+
const runner = new BenchmarkRunner({ simulation: sim });
|
|
19
|
+
|
|
20
|
+
const split: EvaluationSplit | 'all' =
|
|
21
|
+
options.split === 'train'
|
|
22
|
+
? 'train'
|
|
23
|
+
: options.split === 'eval'
|
|
24
|
+
? 'eval'
|
|
25
|
+
: options.split === 'verification'
|
|
26
|
+
? 'verification'
|
|
27
|
+
: 'all';
|
|
28
|
+
|
|
29
|
+
let strategies: AgentEvaluationStrategy[] | undefined;
|
|
30
|
+
if (options.strategy) {
|
|
31
|
+
const list = options.strategy.split(',').map((s) => s.trim());
|
|
32
|
+
strategies = list as AgentEvaluationStrategy[];
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const tasks = options.task ? [options.task] : undefined;
|
|
36
|
+
|
|
37
|
+
if (!options.json) {
|
|
38
|
+
const modeLabel = sim
|
|
39
|
+
? c.yellow('[SIMULATION MODE — offline CI, no network]')
|
|
40
|
+
: c.green('[REAL MODE — requires network access]');
|
|
41
|
+
console.log(c.bold(c.cyan('\n=== DocOrbit Real-World Agent Evaluation Benchmark ===')));
|
|
42
|
+
console.log(modeLabel);
|
|
43
|
+
if (!sim) {
|
|
44
|
+
console.log(c.gray(' Context7: real context7-mcp stdio subprocess (context7.com)'));
|
|
45
|
+
console.log(c.gray(' Firecrawl: real /v1/scrape API (api.firecrawl.dev)'));
|
|
46
|
+
console.log(c.gray(' Official Docs Fetch: direct HTTPS fetch to official docs URLs'));
|
|
47
|
+
console.log(c.gray(' DocOrbit: real in-process MCP server\n'));
|
|
48
|
+
}
|
|
49
|
+
console.log(`Split: ${c.yellow(split)} | Strategies: ${strategies ? strategies.join(', ') : 'all'}`);
|
|
50
|
+
console.log(`Artifacts: ${options.output || 'eval-results/'}\n`);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const report = await runner.run({
|
|
54
|
+
split,
|
|
55
|
+
strategies,
|
|
56
|
+
tasks,
|
|
57
|
+
outputDir: options.output,
|
|
58
|
+
verbose: options.verbose,
|
|
59
|
+
simulation: sim,
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
if (options.json) {
|
|
63
|
+
console.log(JSON.stringify(report, null, 2));
|
|
64
|
+
return;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const modeTag = sim ? ' [SIMULATION]' : ' [REAL]';
|
|
68
|
+
console.log(c.bold(c.green(`\nEvaluation Complete${modeTag} — ${report.totalTasks} tasks\n`)));
|
|
69
|
+
|
|
70
|
+
const sDocs = report.byStrategy.agent_official_docs_fetch || report.byStrategy.agent_web_search;
|
|
71
|
+
const sFC = report.byStrategy.agent_firecrawl;
|
|
72
|
+
const sDR = report.byStrategy.agent_docorbit;
|
|
73
|
+
|
|
74
|
+
if (sDocs && sC7 && sDR) {
|
|
75
|
+
const pct = (n: number) => `${(n * 100).toFixed(1)}%`;
|
|
76
|
+
|
|
77
|
+
if (sFC) {
|
|
78
|
+
const hr = '---------------------------------------------------------------------------------------------------------------------';
|
|
79
|
+
console.log(hr);
|
|
80
|
+
console.log(c.bold(`| ${'Metric'.padEnd(32)}| ${'Official Docs Fetch'.padEnd(21)}| ${'Firecrawl'.padEnd(14)}| ${'Context7'.padEnd(14)}| ${'DocOrbit'.padEnd(14)}|`));
|
|
81
|
+
console.log(hr);
|
|
82
|
+
|
|
83
|
+
const row = (label: string, vDocs: string, vFC: string, vC7: string, vDR: string) =>
|
|
84
|
+
`| ${label.padEnd(32)}| ${vDocs.padEnd(21)}| ${vFC.padEnd(14)}| ${vC7.padEnd(14)}| ${vDR.padEnd(14)}|`;
|
|
85
|
+
|
|
86
|
+
console.log(row('Overall Task Success Rate', pct(sDocs.overallSuccessRate), pct(sFC.overallSuccessRate), pct(sC7.overallSuccessRate), pct(sDR.overallSuccessRate)));
|
|
87
|
+
console.log(row('Held-Out Eval Success Rate', pct(sDocs.evalSuccessRate), pct(sFC.evalSuccessRate), pct(sC7.evalSuccessRate), pct(sDR.evalSuccessRate)));
|
|
88
|
+
console.log(row('Correct Version Selection', pct(sDocs.versionAccuracy), pct(sFC.versionAccuracy), pct(sC7.versionAccuracy), pct(sDR.versionAccuracy)));
|
|
89
|
+
console.log(row('Retrieval Precision@k', pct(sDocs.meanPrecision), pct(sFC.meanPrecision), pct(sC7.meanPrecision), pct(sDR.meanPrecision)));
|
|
90
|
+
console.log(row('Retrieval Recall@k', pct(sDocs.meanRecall), pct(sFC.meanRecall), pct(sC7.meanRecall), pct(sDR.meanRecall)));
|
|
91
|
+
console.log(row('Mean Token Ingestion', `~${Math.round(sDocs.meanTokens)}`, `~${Math.round(sFC.meanTokens)}`, `~${Math.round(sC7.meanTokens)}`, `~${Math.round(sDR.meanTokens)}`));
|
|
92
|
+
console.log(row('Context Prep Latency', `~${sDocs.meanLatencyMs.toFixed(0)}ms`, `~${sFC.meanLatencyMs.toFixed(0)}ms`, `~${sC7.meanLatencyMs.toFixed(0)}ms`, `~${sDR.meanLatencyMs.toFixed(0)}ms`));
|
|
93
|
+
console.log(row('Avg Tool Calls per Task', sDocs.meanToolCalls.toFixed(1), sFC.meanToolCalls.toFixed(1), sC7.meanToolCalls.toFixed(1), sDR.meanToolCalls.toFixed(1)));
|
|
94
|
+
console.log(row('AST Verification Catches', '—', '—', '—', String(sDR.totalVerificationCatches)));
|
|
95
|
+
console.log(row('Verification False Positives', '—', '—', '—', String(sDR.falsePositives)));
|
|
96
|
+
console.log(hr);
|
|
97
|
+
} else {
|
|
98
|
+
const hr = '--------------------------------------------------------------------------------------------------';
|
|
99
|
+
console.log(hr);
|
|
100
|
+
console.log(c.bold(`| ${'Metric'.padEnd(36)}| ${'Official Docs Fetch'.padEnd(21)}| ${'Context7'.padEnd(14)}| ${'DocOrbit'.padEnd(14)}|`));
|
|
101
|
+
console.log(hr);
|
|
102
|
+
|
|
103
|
+
const row = (label: string, vDocs: string, vC7: string, vDR: string) =>
|
|
104
|
+
`| ${label.padEnd(36)}| ${vDocs.padEnd(21)}| ${vC7.padEnd(14)}| ${vDR.padEnd(14)}|`;
|
|
105
|
+
|
|
106
|
+
console.log(row('Overall Task Success Rate', pct(sDocs.overallSuccessRate), pct(sC7.overallSuccessRate), pct(sDR.overallSuccessRate)));
|
|
107
|
+
console.log(row('Held-Out Eval Success Rate', pct(sDocs.evalSuccessRate), pct(sC7.evalSuccessRate), pct(sDR.evalSuccessRate)));
|
|
108
|
+
console.log(row('Correct Version Selection', pct(sDocs.versionAccuracy), pct(sC7.versionAccuracy), pct(sDR.versionAccuracy)));
|
|
109
|
+
console.log(row('Retrieval Precision@k', pct(sDocs.meanPrecision), pct(sC7.meanPrecision), pct(sDR.meanPrecision)));
|
|
110
|
+
console.log(row('Retrieval Recall@k', pct(sDocs.meanRecall), pct(sC7.meanRecall), pct(sDR.meanRecall)));
|
|
111
|
+
console.log(row('Mean Token Ingestion', `~${Math.round(sDocs.meanTokens)}`, `~${Math.round(sC7.meanTokens)}`, `~${Math.round(sDR.meanTokens)}`));
|
|
112
|
+
console.log(row('Context Prep Latency', `~${sDocs.meanLatencyMs.toFixed(0)}ms`, `~${sC7.meanLatencyMs.toFixed(0)}ms`, `~${sDR.meanLatencyMs.toFixed(0)}ms`));
|
|
113
|
+
console.log(row('Avg Tool Calls per Task', sDocs.meanToolCalls.toFixed(1), sC7.meanToolCalls.toFixed(1), sDR.meanToolCalls.toFixed(1)));
|
|
114
|
+
console.log(row('AST Verification Catches', '—', '—', String(sDR.totalVerificationCatches)));
|
|
115
|
+
console.log(row('Verification False Positives', '—', '—', String(sDR.falsePositives)));
|
|
116
|
+
console.log(hr);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
if (sDR) {
|
|
120
|
+
console.log(c.bold(c.cyan('\nAST Code Verification (`check_api`) Performance:')));
|
|
121
|
+
console.log(` Total Verification Catches: ${c.bold(String(sDR.totalVerificationCatches))}`);
|
|
122
|
+
console.log(` False Positives on Valid Code: ${c.bold(String(sDR.falsePositives))}`);
|
|
123
|
+
console.log(` Dynamic/Ambiguous Fallback Rate: ${c.bold(pct(sDR.insufficientEvidenceRate))}`);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
if (sim) {
|
|
127
|
+
console.log(c.yellow('\n⚠ Results above are from simulated runners. Run without --simulation for real results.'));
|
|
128
|
+
}
|
|
129
|
+
console.log(c.gray('\n* Official Docs Fetch baseline directly fetches known official documentation URLs over HTTPS rather than querying a search engine.'));
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
console.log(`\nRaw artifacts: ${c.cyan(options.output || 'eval-results/')}\n`);
|
|
133
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { DocOrbitDb, DocOrbitRepository } from '../../../../packages/storage/src/index.ts';
|
|
2
|
+
import { resolveProjectContext } from '../../../../packages/workspace/src/index.ts';
|
|
3
|
+
import type { ExamplesCommandOptions } from '../../../../packages/shared/src/index.ts';
|
|
4
|
+
import { formatIndexedExamples } from '../formatters/terminal.ts';
|
|
5
|
+
|
|
6
|
+
export async function runExamplesCommand(task: string = '', options: ExamplesCommandOptions = {}): Promise<void> {
|
|
7
|
+
const dbPath = options.dbPath || '.docorbit/docorbit.db';
|
|
8
|
+
const db = new DocOrbitDb(dbPath);
|
|
9
|
+
const repository = new DocOrbitRepository(db);
|
|
10
|
+
|
|
11
|
+
try {
|
|
12
|
+
let effectiveDocVersion = options.docVersion;
|
|
13
|
+
|
|
14
|
+
if (options.projectDir) {
|
|
15
|
+
const projContext = resolveProjectContext(options.projectDir, task, repository);
|
|
16
|
+
if (projContext.matchedDependency && !effectiveDocVersion) {
|
|
17
|
+
effectiveDocVersion = projContext.matchedDependency.targetDocVersion;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const examples = repository.searchIndexedExamples(task, {
|
|
22
|
+
language: options.language,
|
|
23
|
+
framework: options.framework,
|
|
24
|
+
docVersion: effectiveDocVersion,
|
|
25
|
+
limit: options.limit || 10,
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
if (options.json) {
|
|
29
|
+
console.log(JSON.stringify(examples, null, 2));
|
|
30
|
+
} else {
|
|
31
|
+
console.log(formatIndexedExamples(examples, task));
|
|
32
|
+
}
|
|
33
|
+
} catch (err: unknown) {
|
|
34
|
+
console.error(`DocOrbit Examples Error: ${err instanceof Error ? err.message : String(err)}`);
|
|
35
|
+
process.exit(1);
|
|
36
|
+
} finally {
|
|
37
|
+
db.close();
|
|
38
|
+
}
|
|
39
|
+
}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import { DocOrbitDb, DocOrbitRepository } from '../../../../packages/storage/src/index.ts';
|
|
3
|
+
import { ExportService } from '../../../../packages/export/src/index.ts';
|
|
4
|
+
import type { ExportFormat } from '../../../../packages/shared/src/index.ts';
|
|
5
|
+
import { c } from '../formatters/colors.ts';
|
|
6
|
+
|
|
7
|
+
export interface ExportCommandOptions {
|
|
8
|
+
format?: string;
|
|
9
|
+
output?: string;
|
|
10
|
+
stdout?: boolean;
|
|
11
|
+
projectDir?: string;
|
|
12
|
+
docVersion?: string;
|
|
13
|
+
targetSource?: string;
|
|
14
|
+
json?: boolean;
|
|
15
|
+
dbPath?: string;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export async function runExportCommand(
|
|
19
|
+
formatArg?: string,
|
|
20
|
+
options: ExportCommandOptions = {}
|
|
21
|
+
): Promise<void> {
|
|
22
|
+
const rawFormat = (formatArg || options.format || 'agents.md').toLowerCase();
|
|
23
|
+
const validFormats: ExportFormat[] = ['agents.md', 'claude.md', 'skill.md', 'llms.txt', 'docs-map.md'];
|
|
24
|
+
|
|
25
|
+
let format: ExportFormat = 'agents.md';
|
|
26
|
+
if (validFormats.includes(rawFormat as ExportFormat)) {
|
|
27
|
+
format = rawFormat as ExportFormat;
|
|
28
|
+
} else if (rawFormat === 'agents' || rawFormat === 'agent') {
|
|
29
|
+
format = 'agents.md';
|
|
30
|
+
} else if (rawFormat === 'claude') {
|
|
31
|
+
format = 'claude.md';
|
|
32
|
+
} else if (rawFormat === 'skill') {
|
|
33
|
+
format = 'skill.md';
|
|
34
|
+
} else if (rawFormat === 'llms' || rawFormat === 'llms.txt') {
|
|
35
|
+
format = 'llms.txt';
|
|
36
|
+
} else if (rawFormat === 'map' || rawFormat === 'docs-map') {
|
|
37
|
+
format = 'docs-map.md';
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const projectDir = options.projectDir ? path.resolve(options.projectDir) : process.cwd();
|
|
41
|
+
const dbPath = options.dbPath || path.join(projectDir, '.docorbit', 'docorbit.db');
|
|
42
|
+
|
|
43
|
+
const db = new DocOrbitDb(dbPath);
|
|
44
|
+
const repo = new DocOrbitRepository(db);
|
|
45
|
+
const service = new ExportService(repo);
|
|
46
|
+
|
|
47
|
+
try {
|
|
48
|
+
if (options.stdout) {
|
|
49
|
+
const result = await service.generateExport({
|
|
50
|
+
format,
|
|
51
|
+
projectDir,
|
|
52
|
+
docVersion: options.docVersion,
|
|
53
|
+
targetSource: options.targetSource,
|
|
54
|
+
stdout: true,
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
if (options.json) {
|
|
58
|
+
console.log(JSON.stringify(result, null, 2));
|
|
59
|
+
} else {
|
|
60
|
+
process.stdout.write(result.content);
|
|
61
|
+
}
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const result = await service.writeExport({
|
|
66
|
+
format,
|
|
67
|
+
projectDir,
|
|
68
|
+
docVersion: options.docVersion,
|
|
69
|
+
targetSource: options.targetSource,
|
|
70
|
+
outputPath: options.output,
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
if (options.json) {
|
|
74
|
+
console.log(JSON.stringify(result, null, 2));
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
console.log(`\n${c.bold(c.green('✓'))} Generated ${c.bold(format)} for coding agents`);
|
|
79
|
+
console.log(` ${c.dim('Output file:')} ${c.cyan(result.outputPath || 'stdout')}`);
|
|
80
|
+
console.log(` ${c.dim('Doc Version:')} ${result.metadata.docVersion || 'global/latest'}`);
|
|
81
|
+
console.log(` ${c.dim('Estimated Tokens:')} ~${result.metadata.tokenEstimate.toLocaleString()}`);
|
|
82
|
+
console.log(` ${c.dim('APIs Included:')} ${result.metadata.totalApis}`);
|
|
83
|
+
console.log(` ${c.dim('Pitfalls Included:')}${result.metadata.totalPitfalls}`);
|
|
84
|
+
console.log(` ${c.dim('Indexed Sources:')} ${result.metadata.sources.length}`);
|
|
85
|
+
console.log(` ${c.dim('Security Boundary:')}${c.yellow('untrusted documentation preserved')}\n`);
|
|
86
|
+
} finally {
|
|
87
|
+
db.close();
|
|
88
|
+
}
|
|
89
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { DocOrbitDb, DocOrbitRepository } from '../../../../packages/storage/src/index.ts';
|
|
2
|
+
import { ImpactAnalysisService } from '../../../../packages/verification/src/index.ts';
|
|
3
|
+
import type { ImpactCommandOptions } from '../../../../packages/shared/src/index.ts';
|
|
4
|
+
import { formatImpactReport } from '../formatters/terminal.ts';
|
|
5
|
+
|
|
6
|
+
export async function runImpactCommand(target: string = '', options: ImpactCommandOptions = {}): Promise<void> {
|
|
7
|
+
const dbPath = options.dbPath || '.docorbit/docorbit.db';
|
|
8
|
+
const db = new DocOrbitDb(dbPath);
|
|
9
|
+
const repo = new DocOrbitRepository(db);
|
|
10
|
+
|
|
11
|
+
try {
|
|
12
|
+
const service = new ImpactAnalysisService(repo);
|
|
13
|
+
const { result } = service.analyzeImpact({
|
|
14
|
+
projectDir: options.projectDir || process.cwd(),
|
|
15
|
+
fromVersion: options.from,
|
|
16
|
+
toVersion: options.to,
|
|
17
|
+
sourceId: options.source || (target && !target.startsWith('-') ? target : undefined),
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
if (options.json) {
|
|
21
|
+
console.log(JSON.stringify(result, null, 2));
|
|
22
|
+
} else {
|
|
23
|
+
console.log(formatImpactReport(result));
|
|
24
|
+
}
|
|
25
|
+
} catch (err: unknown) {
|
|
26
|
+
console.error(`DocOrbit Impact Error: ${err instanceof Error ? err.message : String(err)}`);
|
|
27
|
+
process.exit(1);
|
|
28
|
+
} finally {
|
|
29
|
+
db.close();
|
|
30
|
+
}
|
|
31
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { resolve } from 'node:path';
|
|
2
|
+
import { DocOrbitDb, DocOrbitRepository } from '../../../../packages/storage/src/index.ts';
|
|
3
|
+
import {
|
|
4
|
+
detectWorkspaceDependencies,
|
|
5
|
+
generateDocsLock,
|
|
6
|
+
readDocsLock,
|
|
7
|
+
writeDocsLock,
|
|
8
|
+
} from '../../../../packages/workspace/src/index.ts';
|
|
9
|
+
|
|
10
|
+
export interface InitCommandOptions {
|
|
11
|
+
json?: boolean;
|
|
12
|
+
dbPath?: string;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export async function runInitCommand(
|
|
16
|
+
targetDir: string = '.',
|
|
17
|
+
options: InitCommandOptions = {}
|
|
18
|
+
): Promise<void> {
|
|
19
|
+
const projectDir = resolve(targetDir);
|
|
20
|
+
const dbPath = options.dbPath || '.docorbit/docorbit.db';
|
|
21
|
+
const db = new DocOrbitDb(dbPath);
|
|
22
|
+
const repository = new DocOrbitRepository(db);
|
|
23
|
+
|
|
24
|
+
try {
|
|
25
|
+
const scanResult = detectWorkspaceDependencies(projectDir);
|
|
26
|
+
const existingLock = readDocsLock(projectDir);
|
|
27
|
+
const lock = generateDocsLock(scanResult, repository, existingLock);
|
|
28
|
+
|
|
29
|
+
writeDocsLock(projectDir, lock);
|
|
30
|
+
|
|
31
|
+
if (options.json) {
|
|
32
|
+
console.log(JSON.stringify(lock, null, 2));
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
console.log(`\n=== DocOrbit Project Initialization ===`);
|
|
37
|
+
console.log(`Workspace Root: ${projectDir}`);
|
|
38
|
+
console.log(`Ecosystems: ${scanResult.ecosystems.length > 0 ? scanResult.ecosystems.join(', ') : 'None detected'}`);
|
|
39
|
+
console.log(`Manifests: ${scanResult.manifestsFound.join(', ') || 'None'}`);
|
|
40
|
+
console.log(`Dependencies: ${scanResult.dependencies.length} detected`);
|
|
41
|
+
|
|
42
|
+
const lockedCount = Object.keys(lock.dependencies).length;
|
|
43
|
+
console.log(`Locked Docs: ${lockedCount} dependencies resolved to documentation\n`);
|
|
44
|
+
|
|
45
|
+
if (lockedCount > 0) {
|
|
46
|
+
console.log('-----------------------------------------------------------------------------');
|
|
47
|
+
console.log('| Package | Project Ver | Doc Ver | Match | Confidence |');
|
|
48
|
+
console.log('-----------------------------------------------------------------------------');
|
|
49
|
+
for (const [key, dep] of Object.entries(lock.dependencies)) {
|
|
50
|
+
const pkgPad = key.padEnd(21).slice(0, 21);
|
|
51
|
+
const projPad = dep.resolvedDependencyVersion.padEnd(11).slice(0, 11);
|
|
52
|
+
const docPad = dep.docVersion.padEnd(8).slice(0, 8);
|
|
53
|
+
const matchPad = dep.matchType.padEnd(11).slice(0, 11);
|
|
54
|
+
const confPad = (dep.confidence * 100).toFixed(0) + '%';
|
|
55
|
+
console.log(`| ${pkgPad} | ${projPad} | ${docPad} | ${matchPad} | ${confPad.padEnd(10)} |`);
|
|
56
|
+
}
|
|
57
|
+
console.log('-----------------------------------------------------------------------------');
|
|
58
|
+
console.log(`\nGenerated deterministic docs.lock at: ${projectDir}/docs.lock\n`);
|
|
59
|
+
} else {
|
|
60
|
+
console.log('Note: No ingested documentation matched detected dependencies.');
|
|
61
|
+
console.log('Tip: Use `docorbit add <url>` to ingest official docs for your dependencies, then re-run `docorbit init`.\n');
|
|
62
|
+
}
|
|
63
|
+
} catch (err: unknown) {
|
|
64
|
+
console.error(`DocOrbit Init Error: ${err instanceof Error ? err.message : String(err)}`);
|
|
65
|
+
process.exit(1);
|
|
66
|
+
} finally {
|
|
67
|
+
db.close();
|
|
68
|
+
}
|
|
69
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { inspectDocumentation } from '../../../../packages/core/src/index.ts';
|
|
2
|
+
import { formatInspectionReport } from '../formatters/terminal.ts';
|
|
3
|
+
|
|
4
|
+
export interface InspectCommandOptions {
|
|
5
|
+
json?: boolean;
|
|
6
|
+
allowLocalhost?: boolean;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export async function runInspectCommand(targetUrl: string, options: InspectCommandOptions = {}): Promise<void> {
|
|
10
|
+
if (!targetUrl) {
|
|
11
|
+
console.error('Error: Please provide a documentation target URL.');
|
|
12
|
+
console.error('Usage: docorbit inspect <url> [--json]');
|
|
13
|
+
process.exit(1);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
try {
|
|
17
|
+
const report = await inspectDocumentation(targetUrl, {
|
|
18
|
+
allowLocalhostForTesting: options.allowLocalhost,
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
if (options.json) {
|
|
22
|
+
console.log(JSON.stringify(report, null, 2));
|
|
23
|
+
} else {
|
|
24
|
+
console.log(formatInspectionReport(report));
|
|
25
|
+
}
|
|
26
|
+
} catch (err: unknown) {
|
|
27
|
+
console.error(`DocOrbit Inspection Error: ${err instanceof Error ? err.message : String(err)}`);
|
|
28
|
+
process.exit(1);
|
|
29
|
+
}
|
|
30
|
+
}
|