okfy-ai 0.1.2

package/dist/cli.d.ts

@@ -0,0 +1 @@
+#!/usr/bin/env node

package/dist/cli.js

@@ -0,0 +1,151 @@
+#!/usr/bin/env node
+import {
+  crawlWebsite,
+  importLocal,
+  inspectBundle,
+  serveMcpStdio,
+  validateBundle
+} from "./chunk-C46QXZDU.js";
+
+// src/cli.ts
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+import { Command } from "commander";
+import pc from "picocolors";
+var program = new Command();
+var packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+function collect(value, previous) {
+  previous.push(value);
+  return previous;
+}
+function printValidation(report, json) {
+  if (json) {
+    console.log(JSON.stringify(report, null, 2));
+    return;
+  }
+  console.log(report.valid ? pc.green("OKF bundle valid") : pc.red("OKF bundle invalid"));
+  console.log(`Concepts: ${report.conceptCount}`);
+  for (const item of report.issues) {
+    const color = item.severity === "error" ? pc.red : pc.yellow;
+    console.log(`${color(item.severity.toUpperCase())} ${item.code}${item.path ? ` ${item.path}` : ""}: ${item.message}`);
+  }
+}
+function printStats(stats) {
+  console.log(`Title: ${stats.title}`);
+  console.log(`Concepts: ${stats.conceptCount}`);
+  console.log(`Links: ${stats.linkCount}`);
+  console.log(`Broken links: ${stats.brokenLinks}`);
+  console.log(`Orphans: ${stats.orphanConcepts.length}`);
+  console.log("Types:");
+  for (const [type, count] of Object.entries(stats.typeDistribution)) console.log(`  ${type}: ${count}`);
+  console.log("Top linked concepts:");
+  for (const item of stats.topLinkedConcepts.slice(0, 5)) console.log(`  ${item.id}: ${item.count}`);
+  if (Object.keys(stats.sourceDomains).length) {
+    console.log("Source domains:");
+    for (const [domain, count] of Object.entries(stats.sourceDomains)) console.log(`  ${domain}: ${count}`);
+  }
+}
+program.name("okfy").description("Turn docs into agent memory with Open Knowledge Format and MCP.").version("0.1.0");
+program.command("crawl").argument("<url>", "Docs URL to crawl").requiredOption("--out <dir>", "Output OKF bundle directory").option("--max-pages <n>", "Maximum pages", (value) => Number(value), 100).option("--max-depth <n>", "Maximum crawl depth", (value) => Number(value), 4).option("--include <pattern>", "Include glob or regex", collect, []).option("--exclude <pattern>", "Exclude glob or regex", collect, []).option("--same-origin", "Stay on same origin", true).option("--no-same-origin", "Allow cross-origin links").option("--respect-robots", "Respect robots.txt", true).option("--no-respect-robots", "Ignore robots.txt").option("--concurrency <n>", "Fetch concurrency", (value) => Number(value), 4).option("--title <name>", "Bundle title").option("--force", "Overwrite output directory", false).option("--dry-run", "List pages that would be crawled", false).option("--allow-private-network", "Allow localhost/private IP crawl targets", false).option("--stable-timestamps", "Use a deterministic timestamp in generated frontmatter", false).action(async (url, options) => {
+  try {
+    const result = await crawlWebsite({
+      seedUrl: url,
+      outDir: options.out,
+      ...options,
+      timestamp: options.stableTimestamps ? "2026-06-14T00:00:00.000Z" : void 0
+    });
+    if (options.dryRun) {
+      console.log("okfy crawl dry run");
+      for (const page of result.dryRunPages ?? []) console.log(page);
+      return;
+    }
+    console.log("okfy crawl");
+    console.log(`Seed: ${url}`);
+    console.log(`Pages: ${result.pagesFetched} fetched, ${result.skipped} skipped, ${result.failed} failed`);
+    console.log(`Concepts: ${result.written.length} written`);
+    console.log(`Output: ${options.out}`);
+    console.log("\nNext:");
+    console.log(`  okfy validate ${options.out}`);
+    console.log(`  okfy serve ${options.out} --mcp`);
+  } catch (error) {
+    console.error(pc.red(error?.message ?? "Crawl failed."));
+    process.exitCode = 1;
+  }
+});
+program.command("import").argument("<path>", "Local docs folder or file").requiredOption("--out <dir>", "Output OKF bundle directory").option("--source-name <name>", "Source name").option("--include <glob>", "Include glob", collect, []).option("--exclude <glob>", "Exclude glob", collect, []).option("--force", "Overwrite output directory", false).option("--stable-timestamps", "Use a deterministic timestamp in generated frontmatter", false).action(async (input, options) => {
+  try {
+    const result = await importLocal({
+      inputPath: input,
+      outDir: options.out,
+      ...options,
+      timestamp: options.stableTimestamps ? "2026-06-14T00:00:00.000Z" : void 0
+    });
+    console.log("okfy import");
+    console.log(`Source: ${input}`);
+    console.log(`Concepts: ${result.written.length} written`);
+    console.log(`Output: ${options.out}`);
+  } catch (error) {
+    console.error(pc.red(error?.message ?? "Import failed."));
+    process.exitCode = 1;
+  }
+});
+program.command("validate").argument("<bundle>", "OKF bundle directory").option("--json", "Print JSON report", false).action(async (bundle, options) => {
+  const report = await validateBundle(bundle);
+  printValidation(report, options.json);
+  if (!report.valid) process.exitCode = 1;
+});
+program.command("inspect").argument("<bundle>", "OKF bundle directory").action(async (bundle) => {
+  try {
+    printStats(await inspectBundle(bundle));
+  } catch (error) {
+    console.error(pc.red(error?.message ?? "Inspect failed."));
+    process.exitCode = 1;
+  }
+});
+program.command("serve").argument("<bundle>", "OKF bundle directory").option("--mcp", "Start MCP server", false).option("--transport <transport>", "Transport: stdio", "stdio").option("--name <server-name>", "MCP server name", "okfy").option("--max-result-chars <n>", "Maximum characters per tool result", (value) => Number(value), 12e3).action(async (bundle, options) => {
+  if (!options.mcp) {
+    console.error(pc.red("Only --mcp mode is supported in v0.1."));
+    process.exitCode = 1;
+    return;
+  }
+  if (options.transport !== "stdio") {
+    console.error(pc.red("Only stdio transport is supported in v0.1."));
+    process.exitCode = 1;
+    return;
+  }
+  await serveMcpStdio({ bundleDir: bundle, name: options.name, maxResultChars: options.maxResultChars });
+});
+function resolveDemoBundle() {
+  const relativeBundle = "examples/bundles/okfy-docs";
+  if (fs.existsSync(relativeBundle)) return relativeBundle;
+  return path.join(packageRoot, relativeBundle);
+}
+program.command("demo").description("Run offline demo against committed example bundle").action(async () => {
+  const bundle = resolveDemoBundle();
+  console.log("okfy demo");
+  console.log(`Offline bundle: ${bundle}`);
+  const report = await validateBundle(bundle);
+  printValidation(report, false);
+  if (!report.valid) {
+    process.exitCode = 1;
+    return;
+  }
+  console.log("");
+  printStats(await inspectBundle(bundle));
+  console.log("");
+  console.log("MCP config:");
+  console.log(
+    JSON.stringify(
+      { mcpServers: { "okfy-docs": { command: "npx", args: ["okfy-ai", "serve", bundle, "--mcp"] } } },
+      null,
+      2
+    )
+  );
+  console.log("");
+  console.log("Ask an agent:");
+  console.log("1. Search okfy docs for crawler security defaults, then cite source concepts.");
+  console.log("2. Read the MCP setup concept and explain the stdio config.");
+  console.log("3. Find importer concepts and list supported input formats.");
+});
+program.parseAsync(process.argv);

package/dist/index.d.ts

@@ -0,0 +1,179 @@
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+
+type ContentType = "html" | "markdown" | "mdx" | "text";
+type RawDocument = {
+    sourceId: string;
+    url?: string;
+    filePath?: string;
+    contentType: ContentType;
+    raw: string;
+    discoveredAt: string;
+};
+type NormalizedDocument = {
+    sourceId: string;
+    title: string;
+    markdown: string;
+    resource?: string;
+    sourcePath?: string;
+    outputPath?: string;
+    headings: Array<{
+        depth: number;
+        text: string;
+        slug: string;
+    }>;
+    links: Array<{
+        href: string;
+        text: string;
+    }>;
+    tags: string[];
+    type: string;
+};
+type Concept = {
+    id: string;
+    path: string;
+    frontmatter: Record<string, unknown>;
+    type: string;
+    title?: string;
+    description?: string;
+    resource?: string;
+    tags: string[];
+    body: string;
+};
+type KnowledgeGraph = {
+    concepts: Map<string, Concept>;
+    outbound: Map<string, string[]>;
+    backlinks: Map<string, string[]>;
+};
+type ValidationIssue = {
+    severity: "error" | "warning";
+    code: string;
+    message: string;
+    path?: string;
+};
+type ValidationReport = {
+    valid: boolean;
+    issues: ValidationIssue[];
+    conceptCount: number;
+};
+type BundleStats = {
+    title: string;
+    conceptCount: number;
+    typeDistribution: Record<string, number>;
+    tagDistribution: Record<string, number>;
+    linkCount: number;
+    brokenLinks: number;
+    orphanConcepts: string[];
+    topLinkedConcepts: Array<{
+        id: string;
+        title?: string;
+        count: number;
+    }>;
+    sourceDomains: Record<string, number>;
+};
+
+type CrawlOptions = {
+    seedUrl: string;
+    outDir: string;
+    maxPages?: number;
+    maxDepth?: number;
+    include?: string[];
+    exclude?: string[];
+    sameOrigin?: boolean;
+    respectRobots?: boolean;
+    concurrency?: number;
+    title?: string;
+    force?: boolean;
+    dryRun?: boolean;
+    allowPrivateNetwork?: boolean;
+    timestamp?: string;
+};
+type CrawlResult = {
+    pagesFetched: number;
+    skipped: number;
+    failed: number;
+    written: string[];
+    documents: NormalizedDocument[];
+    dryRunPages?: string[];
+};
+declare function crawlWebsite(options: CrawlOptions): Promise<CrawlResult>;
+
+declare function extractInternalLinks(concept: Concept): string[];
+declare function buildGraph(conceptsByAnyKey: Map<string, Concept>): KnowledgeGraph;
+
+type ImportOptions = {
+    inputPath: string;
+    outDir: string;
+    sourceName?: string;
+    include?: string[];
+    exclude?: string[];
+    force?: boolean;
+    timestamp?: string;
+};
+declare function importLocal(options: ImportOptions): Promise<{
+    written: string[];
+    documents: NormalizedDocument[];
+}>;
+
+type ServeOptions = {
+    bundleDir: string;
+    name?: string;
+    maxResultChars?: number;
+};
+declare function createMcpServer(options: ServeOptions): Promise<Server>;
+declare function serveMcpStdio(options: ServeOptions): Promise<void>;
+
+declare function extractHeadings(markdown: string): Array<{
+    depth: number;
+    text: string;
+    slug: string;
+}>;
+declare function extractMarkdownLinks(markdown: string): Array<{
+    href: string;
+    text: string;
+}>;
+declare function inferType(title: string, sourceId: string, markdown: string): string;
+declare function inferTags(title: string, sourceId: string, headings: Array<{
+    text: string;
+}>): string[];
+declare function normalizeDocument(raw: RawDocument): NormalizedDocument;
+declare function descriptionFromMarkdown(markdown: string): string;
+
+declare function readConceptFile(bundleDir: string, absolutePath: string): Promise<Concept>;
+declare function readBundle(bundleDir: string): Promise<Map<string, Concept>>;
+
+type SearchResult = {
+    id: string;
+    title?: string;
+    type: string;
+    description?: string;
+    tags: string[];
+    resource?: string;
+    snippet: string;
+    score: number;
+};
+declare class BundleSearch {
+    readonly graph: KnowledgeGraph;
+    private readonly index;
+    constructor(conceptsByAnyKey: Map<string, Concept>);
+    static fromBundle(bundleDir: string): Promise<BundleSearch>;
+    search(query: string, options?: {
+        type?: string;
+        tags?: string[];
+        limit?: number;
+    }): SearchResult[];
+    getConcept(idOrPath: string): Concept | undefined;
+}
+
+declare function validateBundle(bundleDir: string): Promise<ValidationReport>;
+declare function inspectBundle(bundleDir: string): Promise<BundleStats>;
+
+type WriteBundleOptions = {
+    outDir: string;
+    title?: string;
+    sourceName?: string;
+    force?: boolean;
+    timestamp?: string;
+};
+declare function writeOkfBundle(docs: NormalizedDocument[], options: WriteBundleOptions): Promise<string[]>;
+
+export { BundleSearch, type BundleStats, type Concept, type ContentType, type CrawlOptions, type CrawlResult, type ImportOptions, type KnowledgeGraph, type NormalizedDocument, type RawDocument, type SearchResult, type ServeOptions, type ValidationIssue, type ValidationReport, type WriteBundleOptions, buildGraph, crawlWebsite, createMcpServer, descriptionFromMarkdown, extractHeadings, extractInternalLinks, extractMarkdownLinks, importLocal, inferTags, inferType, inspectBundle, normalizeDocument, readBundle, readConceptFile, serveMcpStdio, validateBundle, writeOkfBundle };

package/dist/index.js

@@ -0,0 +1,40 @@
+import {
+  BundleSearch,
+  buildGraph,
+  crawlWebsite,
+  createMcpServer,
+  descriptionFromMarkdown,
+  extractHeadings,
+  extractInternalLinks,
+  extractMarkdownLinks,
+  importLocal,
+  inferTags,
+  inferType,
+  inspectBundle,
+  normalizeDocument,
+  readBundle,
+  readConceptFile,
+  serveMcpStdio,
+  validateBundle,
+  writeOkfBundle
+} from "./chunk-C46QXZDU.js";
+export {
+  BundleSearch,
+  buildGraph,
+  crawlWebsite,
+  createMcpServer,
+  descriptionFromMarkdown,
+  extractHeadings,
+  extractInternalLinks,
+  extractMarkdownLinks,
+  importLocal,
+  inferTags,
+  inferType,
+  inspectBundle,
+  normalizeDocument,
+  readBundle,
+  readConceptFile,
+  serveMcpStdio,
+  validateBundle,
+  writeOkfBundle
+};

package/docs/mcp-clients.md

@@ -0,0 +1,278 @@
+# MCP Client Setup
+
+okfy exposes OKF bundles through a local stdio MCP server:
+
+```bash
+npx -y okfy-ai serve ./tmp/okfy-docs --mcp
+```
+
+Use stdio for local bundles. MCP stdio means the client launches `okfy` as a subprocess, sends JSON-RPC messages on stdin, and reads JSON-RPC responses on stdout. okfy logs should go to stderr.
+
+## Prepare a Bundle
+
+Offline fixture:
+
+```bash
+npx -y okfy-ai import ./examples/local-markdown --out ./tmp/okfy-docs --force
+npx -y okfy-ai validate ./tmp/okfy-docs
+npx -y okfy-ai inspect ./tmp/okfy-docs
+```
+
+Expected output:
+
+```text
+Concepts: 9
+Validation: valid
+Broken links: 0
+```
+
+Docs-site crawl:
+
+```bash
+npx -y okfy-ai crawl https://docs.stripe.com/checkout --out ./stripe-checkout-okf --max-pages 25
+npx -y okfy-ai validate ./stripe-checkout-okf
+npx -y okfy-ai serve ./stripe-checkout-okf --mcp
+```
+
+## Claude Code
+
+Add okfy as a local stdio server:
+
+```bash
+claude mcp add --transport stdio okfy-docs -- npx -y okfy-ai serve ./tmp/okfy-docs --mcp
+claude mcp list
+```
+
+Open Claude Code and check status:
+
+```text
+/mcp
+```
+
+Project-scoped config, saved as `.mcp.json` at project root:
+
+```json
+{
+  "mcpServers": {
+    "okfy-docs": {
+      "type": "stdio",
+      "command": "npx",
+      "args": ["-y", "okfy-ai", "serve", "./tmp/okfy-docs", "--mcp"]
+    }
+  }
+}
+```
+
+Example prompt:
+
+```text
+Use the okfy-docs MCP server. Search for local Markdown import, read the best matching concept, inspect its neighbors, then explain the workflow with citations.
+```
+
+Expected tool-call sequence:
+
+```text
+ToolSearch or MCP server discovery
+bundle_summary
+search_concepts({ "query": "local Markdown import", "limit": 5 })
+read_concept({ "id": "<best-result-id>" })
+get_neighbors({ "id": "<best-result-id>", "depth": 1 })
+read_concept({ "id": "<neighbor-id>", "max_chars": 4000 })
+final answer with cited resource fields
+```
+
+Troubleshooting:
+
+- `spawn npx ENOENT`: install Node.js >=20 and ensure `npx` is on `PATH`.
+- Server pending: run `/mcp`; approve project-scoped `.mcp.json` if prompted.
+- Empty tools: run `npx -y okfy-ai validate ./tmp/okfy-docs`; invalid bundles should fail before serving.
+- Output too large: lower `--max-result-chars`, or ask the agent to search before reading concepts.
+- Wrong bundle path: use an absolute path in config if client starts from another working directory.
+
+## Claude Desktop
+
+Claude Desktop uses MCP server JSON. Add this entry to `claude_desktop_config.json`:
+
+```json
+{
+  "mcpServers": {
+    "okfy-docs": {
+      "command": "npx",
+      "args": ["-y", "okfy-ai", "serve", "/absolute/path/to/okfy/tmp/okfy-docs", "--mcp"]
+    }
+  }
+}
+```
+
+Exact command represented by the config:
+
+```bash
+npx -y okfy-ai serve /absolute/path/to/okfy/tmp/okfy-docs --mcp
+```
+
+Restart Claude Desktop after editing config.
+
+Example prompt:
+
+```text
+Use okfy-docs. Find concepts about MCP tools, read the relevant concept, then tell me which tool to call first when answering a docs question.
+```
+
+Expected tool-call sequence:
+
+```text
+bundle_summary
+search_concepts({ "query": "MCP tools" })
+read_concept({ "id": "<mcp-tools-id>" })
+answer with cited resource fields
+```
+
+Troubleshooting:
+
+- Desktop cannot find `npx`: replace `"command": "npx"` with full path from `which npx`.
+- Server exits immediately: run exact command in terminal and fix bundle validation errors.
+- No okfy tools visible: restart Claude Desktop after config changes.
+- Relative path fails: use absolute bundle path.
+
+## Codex
+
+Codex supports stdio MCP servers through `config.toml`.
+
+User config path:
+
+```text
+~/.codex/config.toml
+```
+
+Trusted project config path:
+
+```text
+.codex/config.toml
+```
+
+Add:
+
+```toml
+[mcp_servers.okfy_docs]
+command = "npx"
+args = ["-y", "okfy-ai", "serve", "./tmp/okfy-docs", "--mcp"]
+startup_timeout_sec = 20
+tool_timeout_sec = 60
+enabled = true
+```
+
+Exact command represented by the config:
+
+```bash
+npx -y okfy-ai serve ./tmp/okfy-docs --mcp
+```
+
+CLI alternative:
+
+```bash
+codex mcp add okfy_docs -- npx -y okfy-ai serve ./tmp/okfy-docs --mcp
+codex mcp --help
+```
+
+In Codex TUI, inspect active servers:
+
+```text
+/mcp
+```
+
+Example prompt:
+
+```text
+Use the okfy_docs MCP server. Search for the concept about progressive disclosure, read it, then explain how okfy keeps agent context small.
+```
+
+Expected tool-call sequence:
+
+```text
+MCP server initialization
+bundle_summary
+search_concepts({ "query": "progressive disclosure agent context" })
+read_concept({ "id": "<best-result-id>", "max_chars": 6000 })
+get_neighbors({ "id": "<best-result-id>", "depth": 1 })
+final answer with citations
+```
+
+Troubleshooting:
+
+- Config ignored: project `.codex/config.toml` loads only for trusted projects; use user config if unsure.
+- Server startup timeout: increase `startup_timeout_sec` if first `npx` install is slow.
+- Tool timeout: increase `tool_timeout_sec` for large bundles.
+- Relative path wrong: set `cwd` or use absolute bundle path.
+- Need current server list: run `/mcp` in TUI.
+
+## Generic MCP stdio
+
+Use this JSON for clients that accept Claude-style `mcpServers` config:
+
+```json
+{
+  "mcpServers": {
+    "okfy-docs": {
+      "command": "npx",
+      "args": ["-y", "okfy-ai", "serve", "./tmp/okfy-docs", "--mcp"],
+      "env": {}
+    }
+  }
+}
+```
+
+Exact command:
+
+```bash
+npx -y okfy-ai serve ./tmp/okfy-docs --mcp
+```
+
+Expected protocol flow:
+
+```text
+client starts subprocess
+client sends initialize
+server returns capabilities and instructions
+client sends initialized
+client calls tools/list
+agent calls bundle_summary
+agent calls search_concepts
+agent calls read_concept
+agent optionally calls get_neighbors
+agent answers with resource citations
+```
+
+Example prompt:
+
+```text
+Use okfy-docs. Search for OKF bundle structure, read the most relevant concepts, and explain the generated files.
+```
+
+Troubleshooting:
+
+- stdout has logs: okfy must write only MCP JSON-RPC messages to stdout; logs belong on stderr.
+- Client cannot start process: use absolute `command` path and absolute bundle path.
+- `tools/list` empty: confirm `okfy serve` was started with `--mcp`.
+- Search returns weak matches: run `okfy inspect` and verify titles, descriptions, and tags were generated.
+- Agent reads too much: ask it to call `search_concepts` first and `read_concept` with `max_chars`.
+
+## Available okfy MCP Tools
+
+```text
+search_concepts(query, type?, tags?, limit?)
+read_concept(id, max_chars?)
+get_neighbors(id, depth?)
+list_types()
+list_tags()
+bundle_summary()
+```
+
+Recommended answering pattern:
+
+```text
+1. Start with bundle_summary for scope.
+2. Use search_concepts for discovery.
+3. Read only top matching concepts.
+4. Use get_neighbors when relationship context matters.
+5. Cite resource fields from read_concept output.
+```