@tikoci/rosetta 0.2.1 → 0.3.0

package/README.md

@@ -10,7 +10,7 @@ Most retrieval-augmented generation (RAG) systems use vector embeddings to searc
 
 For structured technical documentation like RouterOS, full-text search with [BM25 ranking](https://www.sqlite.org/fts5.html#the_bm25_function) beats vector similarity. Technical terms like "dhcp-snooping" or "/ip/firewall/filter" are exact tokens — [porter stemming](https://www.sqlite.org/fts5.html#porter_tokenizer) and proximity matching handle the rest. No embedding pipeline, no vector database, no API keys. Just a single SQLite file that searches in milliseconds.
 
-The data flows: **HTML docs → SQLite extraction → FTS5 indexes → MCP tools → your AI assistant.** The database is built once from MikroTik's official Confluence documentation export, then the MCP server exposes 11 search tools over stdio transport.
+The data flows: **HTML docs → SQLite extraction → FTS5 indexes → MCP tools → your AI assistant.** The database is built once from MikroTik's official Confluence documentation export, then the MCP server exposes 11 search tools over stdio or HTTP transport.
 
 ## What's Inside
 
@@ -24,6 +24,28 @@ The data flows: **HTML docs → SQLite extraction → FTS5 indexes → MCP tools
 
 ## Quick Start
 
+## MCP Discovery Status
+
+- GitHub MCP Registry listing: planned
+- Official MCP Registry publication: metadata is now prepared in `server.json`
+
+Local install remains the primary path today (`bunx @tikoci/rosetta`).
+
+When ready to publish to the official registry:
+
+```sh
+brew install mcp-publisher
+mcp-publisher validate server.json
+mcp-publisher login github
+mcp-publisher publish server.json
+```
+
+After publication, the server should be discoverable via:
+
+```sh
+curl "https://registry.modelcontextprotocol.io/v0.1/servers?search=io.github.tikoci/rosetta"
+```
+
 ### Option A: Install with Bun (recommended)
 
 Zero install, zero config, no binary signing issues. Requires [Bun](https://bun.sh/) — no Gatekeeper or SmartScreen warnings since there's no compiled binary to sign.
@@ -216,6 +238,67 @@ The AI assistant typically starts with `routeros_search`, then drills into speci
 | **Windows SmartScreen warning** | Use `bunx` install (no SmartScreen issues), or click **More info → Run anyway** |
 | **How to update** | `bunx` always uses the latest published version. For binaries, re-download from [Releases](https://github.com/tikoci/rosetta/releases/latest). |
 
+## HTTP Transport
+
+Most MCP clients use stdio (the default). Some — like the OpenAI platform and remote/LAN setups — require an HTTP endpoint instead. Rosetta supports the [MCP Streamable HTTP transport](https://modelcontextprotocol.io/specification/2025-03-26/basic/transports#streamable-http) via the `--http` flag:
+
+```sh
+rosetta --http                    # http://localhost:8080/mcp
+rosetta --http --port 9090        # custom port
+rosetta --http --host 0.0.0.0    # accessible from LAN
+```
+
+Then point your MCP client at the URL:
+
+```json
+{ "url": "http://localhost:8080/mcp" }
+```
+
+**Key facts:**
+
+- **Read-only** — the server queries a local SQLite database. It does not store data, accept uploads, or modify anything.
+- **No authentication** — designed for local/trusted-network use. For public exposure, put it behind a reverse proxy (nginx, caddy) with TLS and auth.
+- **TLS built-in** — for direct HTTPS without a proxy: `--tls-cert cert.pem --tls-key key.pem` (or `TLS_CERT_PATH` + `TLS_KEY_PATH` env vars)
+- **Defaults to localhost** — binding to all interfaces (`--host 0.0.0.0`) requires an explicit flag and logs a warning.
+- **Origin validation** — rejects cross-origin requests to prevent DNS rebinding attacks.
+- **Stdio remains default** — `--http` is opt-in. Existing stdio configs are unaffected.
+
+The `PORT`, `HOST`, `TLS_CERT_PATH`, and `TLS_KEY_PATH` environment variables are supported (lower precedence than CLI flags).
+
+## Container Images
+
+Release CI publishes multi-arch OCI images to:
+
+- Docker Hub: `ammo74/rosetta`
+- GHCR: `ghcr.io/tikoci/rosetta`
+
+Tags per release:
+
+- `${version}` (example: `v0.2.1`)
+- `latest`
+- `sha-<12-char-commit>`
+
+Container defaults:
+
+- Starts in HTTP mode (`--http`) on `0.0.0.0`
+- Uses `PORT` if set, otherwise 8080
+- Uses HTTPS only when both `TLS_CERT_PATH` and `TLS_KEY_PATH` are set
+
+Examples:
+
+```sh
+docker run --rm -p 8080:8080 ghcr.io/tikoci/rosetta:latest
+```
+
+```sh
+docker run --rm -p 8443:8443 \
+  -e PORT=8443 \
+  -e TLS_CERT_PATH=/certs/cert.pem \
+  -e TLS_KEY_PATH=/certs/key.pem \
+  -v "$PWD/certs:/certs:ro" \
+  ghcr.io/tikoci/rosetta:latest
+```
+
 ## Building from Source
 
 For contributors or when you have access to the MikroTik HTML documentation export.
@@ -276,11 +359,13 @@ make release VERSION=v0.1.0
 
 This cross-compiles to macOS (arm64 + x64), Windows (x64), and Linux (x64), creates ZIP archives, compresses the database, tags the commit, and creates a GitHub Release with all artifacts.
 
+The release workflow also publishes OCI images to Docker Hub (`ammo74/rosetta`) and GHCR (`ghcr.io/tikoci/rosetta`) using crane (no Docker daemon required in CI).
+
 ## Project Structure
 
 ```text
 src/
-├── mcp.ts                  # MCP server (11 tools, stdio) + CLI dispatch
+├── mcp.ts                  # MCP server (11 tools, stdio + HTTP) + CLI dispatch
 ├── setup.ts                # --setup: DB download + MCP client config
 ├── query.ts                # NL → FTS5 query planner, BM25 ranking
 ├── db.ts                   # SQLite schema, WAL mode, FTS5 triggers
@@ -294,6 +379,7 @@ src/
 
 scripts/
 └── build-release.ts        # Cross-compile + package releases
+└── container-entrypoint.sh # OCI image runtime entrypoint (HTTP default)
 ```
 
 ## Data Sources

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tikoci/rosetta",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "description": "RouterOS documentation as SQLite FTS5 — RAG search + command glossary via MCP",
   "type": "module",
   "license": "MIT",

package/src/mcp.ts

@@ -9,10 +9,19 @@
  *   --setup [--force]  Download database + print MCP client config
  *   --version          Print version
  *   --help             Print usage
+ *   --http             Start with Streamable HTTP transport (instead of stdio)
+ *   --port <N>         HTTP listen port (default: 8080, env: PORT)
+ *   --host <ADDR>      HTTP bind address (default: localhost, env: HOST)
+ *   --tls-cert <PATH>  TLS certificate PEM file (enables HTTPS, env: TLS_CERT_PATH)
+ *   --tls-key <PATH>   TLS private key PEM file (requires --tls-cert, env: TLS_KEY_PATH)
  *   (default)          Start MCP server (stdio transport)
  *
  * Environment variables:
  *   DB_PATH — absolute path to ros-help.db (default: next to binary or project root)
+ *   PORT    — HTTP listen port (lower precedence than --port)
+ *   HOST    — HTTP bind address (lower precedence than --host)
+ *   TLS_CERT_PATH — TLS certificate path (lower precedence than --tls-cert)
+ *   TLS_KEY_PATH  — TLS private key path (lower precedence than --tls-key)
  */
 
 import { resolveVersion } from "./paths.ts";
@@ -23,6 +32,12 @@ const RESOLVED_VERSION = resolveVersion(import.meta.dirname);
 
 const args = process.argv.slice(2);
 
+/** Extract the value following a named flag (e.g. --port 8080 → "8080") */
+function getArg(name: string): string | undefined {
+  const idx = args.indexOf(name);
+  return idx !== -1 && idx + 1 < args.length ? args[idx + 1] : undefined;
+}
+
 if (args.includes("--version") || args.includes("-v")) {
   console.log(`rosetta ${RESOLVED_VERSION}`);
   process.exit(0);
@@ -33,13 +48,24 @@ if (args.includes("--help") || args.includes("-h")) {
   console.log();
   console.log("Usage:");
   console.log("  rosetta              Start MCP server (stdio transport)");
+  console.log("  rosetta --http       Start with Streamable HTTP transport");
   console.log("  rosetta --setup      Download database + print MCP client config");
   console.log("  rosetta --setup --force  Re-download database");
   console.log("  rosetta --version    Print version");
   console.log("  rosetta --help       Print this help");
   console.log();
+  console.log("HTTP options (require --http):");
+  console.log("  --port <N>           Listen port (default: 8080, env: PORT)");
+  console.log("  --host <ADDR>        Bind address (default: localhost, env: HOST)");
+  console.log("  --tls-cert <PATH>    TLS certificate PEM file (env: TLS_CERT_PATH)");
+  console.log("  --tls-key <PATH>     TLS private key PEM file (env: TLS_KEY_PATH)");
+  console.log();
   console.log("Environment:");
   console.log("  DB_PATH  Absolute path to ros-help.db (optional)");
+  console.log("  PORT     HTTP listen port (lower precedence than --port)");
+  console.log("  HOST     HTTP bind address (lower precedence than --host)");
+  console.log("  TLS_CERT_PATH  TLS certificate path (lower precedence than --tls-cert)");
+  console.log("  TLS_KEY_PATH   TLS private key path (lower precedence than --tls-key)");
   process.exit(0);
 }
 
@@ -54,8 +80,9 @@ if (args.includes("--setup")) {
 
 // ── MCP Server ──
 
+const useHttp = args.includes("--http");
+
 const { McpServer } = await import("@modelcontextprotocol/sdk/server/mcp.js");
-const { StdioServerTransport } = await import("@modelcontextprotocol/sdk/server/stdio.js");
 const { z } = await import("zod/v3");
 
 // Dynamic imports — db.ts eagerly opens the DB file on import,
@@ -175,19 +202,17 @@ Use after routeros_search identifies a relevant page. Pass the numeric page ID
 Returns: plain text, code blocks, and callout blocks (notes, warnings, info, tips).
 Callouts contain crucial caveats and edge-case details — always review them.
 
-**Large page handling:** Always set max_length (e.g., 30000) on the first call.
-Some pages are 100K+ chars. When max_length is set and the page exceeds it,
+**Large page handling:** max_length defaults to 16000. When page content exceeds it,
 pages with sections return a **table of contents** instead of truncated text.
 The TOC lists each section's heading, hierarchy level, character count, and
 deep-link URL. Re-call with the section parameter to retrieve specific sections.
 
 **Section parameter:** Pass a section heading or anchor_id (from the TOC)
-to get that section's content. Parent sections automatically include all
-sub-section content, so requesting a top-level heading gives you everything
-under it.
+to get that section's content. If a section is still too large, its sub-section
+TOC is returned instead — request a more specific sub-section.
 
 Recommended workflow for large pages:
-1. Call with max_length=30000 → get TOC if page is large
+1. First call → get TOC if page is large (automatic with default max_length)
 2. Review section headings to find the relevant section
 3. Call again with section="Section Name" to get that section's content
 
@@ -204,12 +229,12 @@ Workflow — what to do with this content:
         .number()
         .int()
         .min(1000)
-        .optional()
-        .describe("Recommended: set to 30000. Max combined text+code length. If exceeded and page has sections, returns a TOC instead of truncated text. Omit only if you need the entire page."),
+        .default(16000)
+        .describe("Max combined text+code length (default: 16000). If exceeded and page has sections, returns a TOC instead of truncated text. Set higher (e.g. 50000) to get more content in one call."),
       section: z
         .string()
         .optional()
-        .describe("Section heading or anchor_id from TOC. Returns only that section's content."),
+        .describe("Section heading or anchor_id from TOC. Returns only that section's content (also subject to max_length)."),
     },
   },
   async ({ page, max_length, section }) => {
@@ -739,7 +764,94 @@ Requires network access to upgrade.mikrotik.com.`,
 
 // ---- Start ----
 
-const transport = new StdioServerTransport();
-await server.connect(transport);
+if (useHttp) {
+  const { existsSync } = await import("node:fs");
+  const { WebStandardStreamableHTTPServerTransport } = await import(
+    "@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js"
+  );
+
+  const port = Number(getArg("--port") ?? process.env.PORT ?? 8080);
+  const hostname = getArg("--host") ?? process.env.HOST ?? "localhost";
+  const tlsCert = getArg("--tls-cert") ?? process.env.TLS_CERT_PATH;
+  const tlsKey = getArg("--tls-key") ?? process.env.TLS_KEY_PATH;
+
+  if ((tlsCert && !tlsKey) || (!tlsCert && tlsKey)) {
+    process.stderr.write(
+      "Error: TLS cert and key must both be provided (via flags or TLS_CERT_PATH/TLS_KEY_PATH)\n"
+    );
+    process.exit(1);
+  }
+  if (tlsCert && !existsSync(tlsCert)) {
+    process.stderr.write(`Error: TLS certificate not found: ${tlsCert}\n`);
+    process.exit(1);
+  }
+  if (tlsKey && !existsSync(tlsKey)) {
+    process.stderr.write(`Error: TLS private key not found: ${tlsKey}\n`);
+    process.exit(1);
+  }
+
+  const useTls = !!(tlsCert && tlsKey);
+  const scheme = useTls ? "https" : "http";
+
+  const httpTransport = new WebStandardStreamableHTTPServerTransport({
+    sessionIdGenerator: () => crypto.randomUUID(),
+    enableJsonResponse: false,
+  });
+
+  await server.connect(httpTransport);
+
+  const isLAN = hostname === "0.0.0.0" || hostname === "::";
+  if (isLAN) {
+    process.stderr.write(
+      "Warning: Binding to all interfaces — the MCP server will be accessible from the network.\n"
+    );
+    if (!useTls) {
+      process.stderr.write(
+        "  Consider using --tls-cert/--tls-key or a reverse proxy for production use.\n"
+      );
+    }
+  }
+
+  Bun.serve({
+    port,
+    hostname,
+    ...(useTls && tlsCert && tlsKey
+      ? { tls: { cert: Bun.file(tlsCert), key: Bun.file(tlsKey) } }
+      : {}),
+    async fetch(req: Request): Promise<Response> {
+      const url = new URL(req.url);
+
+      // DNS rebinding protection: reject browser-origin requests
+      const origin = req.headers.get("origin");
+      if (origin) {
+        try {
+          const originHost = new URL(origin).host;
+          const serverHost = `${hostname === "0.0.0.0" || hostname === "::" ? "localhost" : hostname}:${port}`;
+          if (originHost !== serverHost && originHost !== `localhost:${port}` && originHost !== `127.0.0.1:${port}`) {
+            return new Response("Forbidden: Origin not allowed", { status: 403 });
+          }
+        } catch {
+          return new Response("Forbidden: Invalid Origin", { status: 403 });
+        }
+      }
+
+      if (url.pathname === "/mcp") {
+        return httpTransport.handleRequest(req);
+      }
+
+      return new Response("Not Found", { status: 404 });
+    },
+  });
+
+  const displayHost = isLAN ? "localhost" : hostname;
+  process.stderr.write(`rosetta ${RESOLVED_VERSION} — Streamable HTTP\n`);
+  process.stderr.write(`  ${scheme}://${displayHost}:${port}/mcp\n`);
+} else {
+  const { StdioServerTransport } = await import(
+    "@modelcontextprotocol/sdk/server/stdio.js"
+  );
+  const transport = new StdioServerTransport();
+  await server.connect(transport);
+}
 
 })();

package/src/query.ts

@@ -184,6 +184,7 @@ export function getPage(idOrTitle: string | number, maxLength?: number, section?
   word_count: number;
   code_lines: number;
   callouts: Array<{ type: string; content: string }>;
+  callout_summary?: { count: number; types: Record<string, number> };
   truncated?: { text_total: number; code_total: number };
   sections?: SectionTocEntry[];
   section?: { heading: string; level: number; anchor_id: string };
@@ -221,11 +222,11 @@ export function getPage(idOrTitle: string | number, maxLength?: number, section?
 
       const descendants = db
         .prepare(
-          `SELECT heading, level, text, code, word_count
+          `SELECT heading, level, anchor_id, text, code, word_count
            FROM sections WHERE page_id = ? AND sort_order > ? AND level > ? AND sort_order < ?
            ORDER BY sort_order`,
         )
-        .all(page.id, sec.sort_order, sec.level, upperBound) as Array<{ heading: string; level: number; text: string; code: string; word_count: number }>;
+        .all(page.id, sec.sort_order, sec.level, upperBound) as Array<{ heading: string; level: number; anchor_id: string; text: string; code: string; word_count: number }>;
 
       let fullText = sec.text;
       let fullCode = sec.code;
@@ -237,6 +238,34 @@ export function getPage(idOrTitle: string | number, maxLength?: number, section?
         totalWords += child.word_count;
       }
 
+      // If section+descendants exceed maxLength and there are subsections, return sub-TOC
+      if (maxLength && (fullText.length + fullCode.length) > maxLength && descendants.length > 0) {
+        const subToc: SectionTocEntry[] = [
+          { heading: sec.heading, level: sec.level, anchor_id: sec.anchor_id, char_count: sec.text.length + sec.code.length, url: `${page.url}#${sec.anchor_id}` },
+          ...descendants.map((d) => ({ heading: d.heading, level: d.level, anchor_id: d.anchor_id, char_count: d.text.length + d.code.length, url: `${page.url}#${d.anchor_id}` })),
+        ];
+        const totalChars = fullText.length + fullCode.length;
+        return {
+          id: page.id, title: page.title, path: page.path, url: `${page.url}#${sec.anchor_id}`,
+          text: "", code: "",
+          word_count: totalWords, code_lines: 0,
+          callouts: [], callout_summary: calloutSummary(callouts),
+          sections: subToc,
+          section: { heading: sec.heading, level: sec.level, anchor_id: sec.anchor_id },
+          note: `Section "${sec.heading}" content (${totalChars} chars) exceeds max_length (${maxLength}). Showing ${subToc.length} sub-sections. Re-call with a more specific section heading or anchor_id.`,
+        };
+      }
+
+      // If section exceeds maxLength but has no sub-sections, truncate
+      if (maxLength && (fullText.length + fullCode.length) > maxLength) {
+        const textTotal = fullText.length;
+        const codeTotal = fullCode.length;
+        const codeBudget = Math.min(codeTotal, Math.floor(maxLength * 0.2));
+        const textBudget = maxLength - codeBudget;
+        fullText = `${fullText.slice(0, textBudget)}\n\n[... truncated — ${textTotal} chars total, showing first ${textBudget}]`;
+        fullCode = codeTotal > codeBudget ? `${fullCode.slice(0, codeBudget)}\n# [... truncated — ${codeTotal} chars total]` : fullCode;
+      }
+
       return {
         id: page.id,
         title: page.title,
@@ -258,7 +287,8 @@ export function getPage(idOrTitle: string | number, maxLength?: number, section?
         id: page.id, title: page.title, path: page.path, url: page.url,
         text: "", code: "",
         word_count: page.word_count, code_lines: page.code_lines,
-        callouts, sections: toc,
+        callouts: [], callout_summary: calloutSummary(callouts),
+        sections: toc,
         note: `Section "${section}" not found. ${toc.length} sections available — use a heading or anchor_id from the list.`,
       };
     }
@@ -284,7 +314,8 @@ export function getPage(idOrTitle: string | number, maxLength?: number, section?
         id: page.id, title: page.title, path: page.path, url: page.url,
         text: "", code: "",
         word_count: page.word_count, code_lines: page.code_lines,
-        callouts, sections: toc,
+        callouts: [], callout_summary: calloutSummary(callouts),
+        sections: toc,
         truncated: { text_total: text.length, code_total: code.length },
         note: `Page content (${totalChars} chars) exceeds max_length (${maxLength}). Showing table of contents with ${toc.length} sections. Re-call with section parameter to retrieve specific sections.`,
       };
@@ -303,6 +334,13 @@ export function getPage(idOrTitle: string | number, maxLength?: number, section?
   return { id: page.id, title: page.title, path: page.path, url: page.url, text, code, word_count: page.word_count, code_lines: page.code_lines, callouts, ...(truncated ? { truncated } : {}) };
 }
 
+/** Build compact callout summary (count + type breakdown) for TOC-mode responses. */
+function calloutSummary(callouts: Array<{ type: string; content: string }>): { count: number; types: Record<string, number> } {
+  const types: Record<string, number> = {};
+  for (const c of callouts) types[c.type] = (types[c.type] || 0) + 1;
+  return { count: callouts.length, types };
+}
+
 /** Build section TOC for a page. */
 function getPageToc(pageId: number, pageUrl: string): SectionTocEntry[] {
   const rows = db

package/src/setup.ts

@@ -167,6 +167,8 @@ function printCompiledConfig(serverCmd: string) {
   console.log("▸ OpenAI Codex");
   console.log(`  codex mcp add rosetta -- ${serverCmd}`);
   console.log();
+
+  printHttpConfig(`${serverCmd} --http`);
 }
 
 function printPackageConfig() {
@@ -228,6 +230,8 @@ function printPackageConfig() {
   console.log("▸ OpenAI Codex");
   console.log(`  codex mcp add rosetta -- bunx @tikoci/rosetta`);
   console.log();
+
+  printHttpConfig("bunx @tikoci/rosetta --http");
 }
 
 function printDevConfig(baseDir: string) {
@@ -276,6 +280,27 @@ function printDevConfig(baseDir: string) {
   console.log("▸ OpenAI Codex");
   console.log(`  codex mcp add rosetta -- bun run src/mcp.ts`);
   console.log();
+
+  printHttpConfig(`bun run src/mcp.ts --http`);
+}
+
+function printHttpConfig(startCmd: string) {
+  console.log("─".repeat(60));
+  console.log("Streamable HTTP transport (for HTTP-only MCP clients):");
+  console.log("─".repeat(60));
+  console.log();
+  console.log("▸ Start in HTTP mode");
+  console.log(`  ${startCmd}`);
+  console.log(`  ${startCmd} --port 9090`);
+  console.log(`  ${startCmd} --host 0.0.0.0          # LAN access`);
+  console.log(`  ${startCmd} --tls-cert cert.pem --tls-key key.pem  # HTTPS`);
+  console.log();
+  console.log("▸ URL-based MCP clients (OpenAI, etc.)");
+  console.log(`  { "url": "http://localhost:8080/mcp" }`);
+  console.log();
+  console.log("  For LAN access, replace localhost with the server's IP address.");
+  console.log("  Use a reverse proxy (nginx, caddy) for production HTTPS.");
+  console.log();
 }
 
 // Run directly