football-docs 0.1.2 → 0.2.0

package/dist/discover.js

@@ -0,0 +1,121 @@
+/**
+ * Source discovery for football data providers.
+ *
+ * Given a provider's known URLs, discovers the best documentation source
+ * to crawl. Checks for llms.txt, ReadTheDocs structure, GitHub docs, etc.
+ *
+ * Usage:
+ *   import { discoverBestSource } from "./discover.js";
+ *   const result = await discoverBestSource(provider);
+ */
+import { fetchText, urlExists } from "./http.js";
+/**
+ * Probe a base URL for llms.txt variants.
+ * Returns the URL and content of the best llms.txt found, or null.
+ */
+async function probeLlmsTxt(baseUrl) {
+    const root = baseUrl.replace(/\/$/, "");
+    for (const filename of ["llms-full.txt", "llms.txt"]) {
+        const url = `${root}/${filename}`;
+        const content = await fetchText(url);
+        if (content && content.length > 100) {
+            return { url, content };
+        }
+    }
+    return null;
+}
+/**
+ * Check if a URL looks like a ReadTheDocs or Sphinx site.
+ * Uses a single GET request (no redundant HEAD).
+ */
+async function probeReadTheDocs(url) {
+    const html = await fetchText(url, 10000);
+    if (!html)
+        return false;
+    return (html.includes("readthedocs") ||
+        html.includes("sphinx") ||
+        html.includes("Read the Docs") ||
+        /\/en\/(latest|stable)\//.test(url));
+}
+/**
+ * Choose the best source to crawl from probe results.
+ *
+ * Scoring: llms-full.txt (>1000 chars) = 100, llms.txt (>1000 chars) = 90,
+ * ReadTheDocs/Sphinx = 70, short llms.txt = 50, reachable = 40, unreachable = 0.
+ *
+ * @param probeResults - What was actually found at each URL
+ * @returns The best source to crawl, or null if only curated content is available
+ */
+export function chooseBestSource(probeResults) {
+    let bestScore = 0;
+    let bestSource = null;
+    for (const probe of probeResults) {
+        let score = 0;
+        if (probe.llmsTxt) {
+            const isFullTxt = probe.llmsTxt.url.includes("llms-full.txt");
+            const isSubstantial = probe.llmsTxt.content.length > 1000;
+            score = isFullTxt && isSubstantial ? 100
+                : isSubstantial ? 90
+                    : 50;
+        }
+        else if (probe.isReadTheDocs) {
+            score = 70;
+        }
+        else if (probe.isReachable) {
+            score = 40;
+        }
+        if (score > bestScore) {
+            bestScore = score;
+            bestSource = probe.llmsTxt
+                ? { url: probe.llmsTxt.url, type: "llms_txt" }
+                : { url: probe.url, type: probe.sourceType };
+        }
+    }
+    return bestSource;
+}
+// ── Public API ───────────────────────────────────────────────────────
+/**
+ * Discover the best documentation source for a provider.
+ *
+ * Probes all configured URLs for llms.txt, ReadTheDocs markers, and
+ * basic reachability, then uses chooseBestSource() to pick the winner.
+ */
+export async function discoverBestSource(provider, sources) {
+    const checked = [];
+    const probeResults = [];
+    for (const source of sources) {
+        if (source.type === "curated" || !source.url)
+            continue;
+        try {
+            const [llmsTxt, isReadTheDocs, reachability] = await Promise.all([
+                probeLlmsTxt(source.url),
+                probeReadTheDocs(source.url),
+                urlExists(source.url),
+            ]);
+            const found = llmsTxt
+                ? `llms.txt (${llmsTxt.content.length} chars)`
+                : isReadTheDocs
+                    ? "readthedocs"
+                    : reachability.ok
+                        ? "reachable"
+                        : null;
+            checked.push({ url: source.url, found });
+            probeResults.push({
+                url: source.url,
+                sourceType: source.type,
+                llmsTxt,
+                isReadTheDocs,
+                isReachable: reachability.ok,
+            });
+        }
+        catch (err) {
+            checked.push({
+                url: source.url,
+                found: null,
+                error: err instanceof Error ? err.message : String(err),
+            });
+        }
+    }
+    const bestSource = chooseBestSource(probeResults);
+    return { provider, source: bestSource, checked };
+}

package/dist/http.d.ts

@@ -0,0 +1,8 @@
+/** Shared HTTP utilities for crawling and discovery. */
+/** Check if a URL returns a successful response. */
+export declare function urlExists(url: string): Promise<{
+    ok: boolean;
+    contentType?: string;
+}>;
+/** Fetch text content from a URL, returning null on failure. Rejects responses over 10MB. */
+export declare function fetchText(url: string, timeoutMs?: number): Promise<string | null>;

package/dist/http.js

@@ -0,0 +1,36 @@
+/** Shared HTTP utilities for crawling and discovery. */
+const MAX_RESPONSE_BYTES = 10 * 1024 * 1024; // 10 MB
+/** Check if a URL returns a successful response. */
+export async function urlExists(url) {
+    try {
+        const response = await fetch(url, {
+            method: "HEAD",
+            signal: AbortSignal.timeout(5000),
+        });
+        return {
+            ok: response.ok,
+            contentType: response.headers.get("content-type") ?? undefined,
+        };
+    }
+    catch {
+        return { ok: false };
+    }
+}
+/** Fetch text content from a URL, returning null on failure. Rejects responses over 10MB. */
+export async function fetchText(url, timeoutMs = 15000) {
+    try {
+        const response = await fetch(url, {
+            signal: AbortSignal.timeout(timeoutMs),
+        });
+        if (!response.ok)
+            return null;
+        // Reject obviously oversized responses early via Content-Length
+        const contentLength = response.headers.get("content-length");
+        if (contentLength && Number(contentLength) > MAX_RESPONSE_BYTES)
+            return null;
+        return await response.text();
+    }
+    catch {
+        return null;
+    }
+}

package/dist/index.d.ts

@@ -2,10 +2,12 @@
  * Nutmeg Football Docs MCP Server
  *
  * A Context7-style searchable index of football data provider documentation.
- * Exposes two tools:
- *   - search_docs: Full-text search across all provider docs
- *   - list_providers: List all indexed providers and their coverage
+ * Exposes tools for searching docs, listing providers, comparing providers,
+ * and requesting documentation updates.
  *
  * Data is stored in a SQLite FTS5 index for fast offline search.
+ * Update requests are stored in a separate writable SQLite DB in a
+ * user-writable location (XDG_DATA_HOME or ~/.local/share).
  */
-export {};
+/** Sanitise a query string for FTS5 MATCH by wrapping in double quotes (phrase search). */
+export declare function sanitiseFtsQuery(query: string): string;

package/dist/index.js

@@ -2,32 +2,70 @@
  * Nutmeg Football Docs MCP Server
  *
  * A Context7-style searchable index of football data provider documentation.
- * Exposes two tools:
- *   - search_docs: Full-text search across all provider docs
- *   - list_providers: List all indexed providers and their coverage
+ * Exposes tools for searching docs, listing providers, comparing providers,
+ * and requesting documentation updates.
  *
  * Data is stored in a SQLite FTS5 index for fast offline search.
+ * Update requests are stored in a separate writable SQLite DB in a
+ * user-writable location (XDG_DATA_HOME or ~/.local/share).
  */
+import { existsSync, mkdirSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
-import { z } from "zod";
 import Database from "better-sqlite3";
-import { resolve, dirname } from "node:path";
-import { fileURLToPath } from "node:url";
-import { existsSync } from "node:fs";
+import { z } from "zod";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const DB_PATH = resolve(__dirname, "..", "data", "docs.db");
+const PKG_VERSION = JSON.parse(readFileSync(resolve(__dirname, "..", "package.json"), "utf-8")).version;
+// Request queue DB goes in a user-writable location, not inside the npm package
+const QUEUE_DB_DIR = resolve(process.env.XDG_DATA_HOME ?? resolve(homedir(), ".local", "share"), "football-docs");
+const QUEUE_DB_PATH = resolve(QUEUE_DB_DIR, "requests.db");
 // ── Database ────────────────────────────────────────────────────────────
 function openDb() {
     if (!existsSync(DB_PATH)) {
         throw new Error(`Docs database not found at ${DB_PATH}. Run 'npm run ingest' first to build the index.`);
     }
-    return new Database(DB_PATH, { readonly: true });
+    const db = new Database(DB_PATH, { readonly: true });
+    // Check schema version — source_type column was added in v0.2.0
+    const columns = db.pragma("table_info(docs)");
+    const hasProvenance = columns.some((c) => c.name === "source_type");
+    if (!hasProvenance) {
+        db.close();
+        throw new Error(`Docs database is outdated (missing provenance columns). Run 'npm run ingest' to rebuild.`);
+    }
+    return db;
+}
+function openQueueDb() {
+    mkdirSync(QUEUE_DB_DIR, { recursive: true });
+    const db = new Database(QUEUE_DB_PATH);
+    db.pragma("journal_mode = WAL");
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS requests (
+      id TEXT PRIMARY KEY,
+      type TEXT NOT NULL,
+      provider TEXT NOT NULL,
+      reason TEXT NOT NULL,
+      suggested_urls TEXT,
+      requested_at TEXT NOT NULL,
+      status TEXT NOT NULL DEFAULT 'pending'
+    )
+  `);
+    return db;
+}
+/** Sanitise a query string for FTS5 MATCH by wrapping in double quotes (phrase search). */
+export function sanitiseFtsQuery(query) {
+    // FTS5 special syntax (AND, OR, NOT, NEAR, column:, *, ^) can cause errors.
+    // Wrap the user query in double quotes to treat it as a phrase search,
+    // escaping any embedded double quotes first.
+    return `"${query.replace(/"/g, '""')}"`;
 }
 // ── Server ──────────────────────────────────────────────────────────────
 const server = new McpServer({
     name: "nutmeg-football-docs",
-    version: "0.1.0",
+    version: PKG_VERSION,
 });
 // Tool: search_docs
 server.tool("search_docs", "Search football data provider documentation. Use for finding event types, qualifier IDs, API endpoints, coordinate systems, data models, and cross-provider mappings. Returns the most relevant documentation chunks.", {
@@ -44,19 +82,22 @@ server.tool("search_docs", "Search football data provider documentation. Use for
 }, async ({ query, provider, max_results }) => {
     const db = openDb();
     try {
+        const safeQuery = sanitiseFtsQuery(query);
         let sql = `
-        SELECT provider, category, title, content,
+        SELECT d.provider, d.category, d.title, d.content,
+               d.source_type, d.source_url, d.upstream_version,
                rank * -1 as relevance
         FROM docs_fts
+        JOIN docs d ON d.id = docs_fts.rowid
         WHERE docs_fts MATCH ?
       `;
-        const params = [query];
+        const params = [safeQuery];
         if (provider) {
-            sql += ` AND provider = ?`;
+            sql += ` AND d.provider = ?`;
             params.push(provider.toLowerCase());
         }
         sql += ` ORDER BY rank LIMIT ?`;
-        params.push(max_results ?? 10);
+        params.push(max_results);
         const rows = db.prepare(sql).all(...params);
         if (rows.length === 0) {
             return {
@@ -69,7 +110,12 @@ server.tool("search_docs", "Search football data provider documentation. Use for
             };
         }
         const results = rows
-            .map((r, i) => `## [${i + 1}] ${r.title}\n**Provider:** ${r.provider} | **Category:** ${r.category}\n\n${r.content}`)
+            .map((r, i) => {
+            const source = r.source_type === "curated"
+                ? "**Source:** curated by football-docs contributors"
+                : `**Source:** ${r.source_type}${r.source_url ? ` (${r.source_url})` : ""}${r.upstream_version ? ` | v${r.upstream_version}` : ""}`;
+            return `## [${i + 1}] ${r.title}\n**Provider:** ${r.provider} | **Category:** ${r.category} | ${source}\n\n${r.content}`;
+        })
             .join("\n\n---\n\n");
         return {
             content: [
@@ -124,19 +170,21 @@ server.tool("compare_providers", "Compare what two or more providers offer for a
 }, async ({ topic, providers }) => {
     const db = openDb();
     try {
+        const safeTopic = sanitiseFtsQuery(topic);
         let sql = `
-        SELECT provider, category, title, content,
+        SELECT d.provider, d.category, d.title, d.content,
                rank * -1 as relevance
         FROM docs_fts
+        JOIN docs d ON d.id = docs_fts.rowid
         WHERE docs_fts MATCH ?
       `;
-        const params = [topic];
+        const params = [safeTopic];
         if (providers && providers.length > 0) {
             const placeholders = providers.map(() => "?").join(", ");
-            sql += ` AND provider IN (${placeholders})`;
+            sql += ` AND d.provider IN (${placeholders})`;
             params.push(...providers.map((p) => p.toLowerCase()));
         }
-        sql += ` ORDER BY provider, rank LIMIT 30`;
+        sql += ` ORDER BY d.provider, rank LIMIT 30`;
         const rows = db.prepare(sql).all(...params);
         if (rows.length === 0) {
             return {
@@ -171,6 +219,66 @@ server.tool("compare_providers", "Compare what two or more providers offer for a
         db.close();
     }
 });
+// Tool: request_update
+server.tool("request_update", "Request that a provider's documentation be added, updated, or recrawled. Use when you notice docs are outdated, a provider is missing, or you know of a better documentation source. Requests are queued for review.", {
+    type: z.enum(["new_provider", "recrawl", "flag_outdated", "suggest_source"]).describe("Type of request: new_provider (add a new tool/library), recrawl (refresh existing docs), flag_outdated (mark docs as stale), suggest_source (recommend a better doc source like llms.txt)"),
+    provider: z.string().max(100).describe("Provider name (existing or proposed). Examples: 'statsbomb', 'mplsoccer', 'floodlight'"),
+    reason: z.string().max(2000).describe("Why this update is needed. Be specific: version bump, missing event types, new API endpoints, etc."),
+    suggested_urls: z.array(z.string().url().max(500)).max(10).optional().describe("URLs for documentation sources (readthedocs, GitHub, llms.txt, etc.)"),
+}, async ({ type, provider, reason, suggested_urls }) => {
+    const qdb = openQueueDb();
+    try {
+        // Hard cap on queue size
+        const countRow = qdb.prepare("SELECT COUNT(*) as cnt FROM requests").get();
+        if (countRow.cnt >= 500) {
+            return {
+                isError: true,
+                content: [
+                    {
+                        type: "text",
+                        text: `Request queue is full (${countRow.cnt} entries). Please file an issue at https://github.com/withqwerty/football-docs/issues instead.`,
+                    },
+                ],
+            };
+        }
+        // Cooldown: reject if same provider was requested in last 7 days
+        const recentRequest = qdb.prepare(`SELECT id, requested_at FROM requests
+         WHERE lower(provider) = lower(?) AND status = 'pending'
+           AND datetime(requested_at) > datetime('now', '-7 days')
+         LIMIT 1`).get(provider);
+        if (recentRequest) {
+            return {
+                isError: true,
+                content: [
+                    {
+                        type: "text",
+                        text: `A pending request for "${provider}" already exists (from ${recentRequest.requested_at}). Requests have a 7-day cooldown to prevent duplicate work. Request ID: ${recentRequest.id}`,
+                    },
+                ],
+            };
+        }
+        const id = `req_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 6)}`;
+        qdb.prepare(`INSERT INTO requests (id, type, provider, reason, suggested_urls, requested_at, status)
+         VALUES (?, ?, ?, ?, ?, ?, 'pending')`).run(id, type, provider.toLowerCase(), reason, suggested_urls ? JSON.stringify(suggested_urls) : null, new Date().toISOString());
+        const typeLabel = {
+            new_provider: "New provider request",
+            recrawl: "Recrawl request",
+            flag_outdated: "Outdated docs flag",
+            suggest_source: "Source suggestion",
+        }[type];
+        return {
+            content: [
+                {
+                    type: "text",
+                    text: `${typeLabel} queued for "${provider}" (ID: ${id}).\n\nReason: ${reason}${suggested_urls?.length ? `\nSuggested URLs:\n${suggested_urls.map((u) => `  - ${u}`).join("\n")}` : ""}\n\nThis request will be reviewed by maintainers. You can also file an issue at https://github.com/withqwerty/football-docs/issues for community visibility.`,
+                },
+            ],
+        };
+    }
+    finally {
+        qdb.close();
+    }
+});
 // ── Start ───────────────────────────────────────────────────────────────
 async function main() {
     const transport = new StdioServerTransport();

package/dist/ingest.d.ts

@@ -1,21 +1,45 @@
 /**
  * Ingest provider documentation into the SQLite FTS5 index.
  *
- * Reads markdown files from docs/providers/{provider}/*.md and chunks them
- * by heading (## or ###), storing each chunk with metadata.
+ * Reads markdown files from docs/{provider}/*.md and chunks them
+ * by heading (## or ###), storing each chunk with provenance metadata.
+ *
+ * Each markdown file can include a YAML frontmatter block with provenance:
+ *   ---
+ *   source_url: https://example.com/docs
+ *   source_type: readthedocs
+ *   upstream_version: 1.8.8
+ *   crawled_at: 2026-03-26T00:00:00Z
+ *   ---
+ *
+ * Files without frontmatter default to source_type: "curated".
  *
  * Usage:
  *   npm run ingest                    # ingest all providers
  *   npm run ingest -- --provider opta # ingest one provider
- *
- * Doc file naming convention:
- *   docs/providers/{provider}/{category}.md
- *
- * Example:
- *   docs/providers/opta/event-types.md
- *   docs/providers/opta/qualifiers.md
- *   docs/providers/opta/coordinate-system.md
- *   docs/providers/statsbomb/events.md
- *   docs/providers/kloppy/data-model.md
  */
+interface Frontmatter {
+    source_url: string | null;
+    source_type: string;
+    upstream_version: string | null;
+    crawled_at: string | null;
+}
+interface DocChunk {
+    provider: string;
+    category: string;
+    title: string;
+    content: string;
+    source_url: string | null;
+    source_type: string;
+    upstream_version: string | null;
+    crawled_at: string | null;
+}
+/** Parse optional YAML frontmatter from a markdown file. */
+export declare function parseFrontmatter(text: string): {
+    frontmatter: Frontmatter;
+    body: string;
+};
+/** Split a markdown file into chunks by ## or ### headings. */
+export declare function chunkMarkdown(text: string, provider: string, category: string): DocChunk[];
+export declare const SCHEMA_SQL = "\n  CREATE TABLE IF NOT EXISTS docs (\n    id INTEGER PRIMARY KEY AUTOINCREMENT,\n    provider TEXT NOT NULL,\n    category TEXT NOT NULL,\n    title TEXT NOT NULL,\n    content TEXT NOT NULL,\n    source_url TEXT,\n    source_type TEXT NOT NULL DEFAULT 'curated',\n    upstream_version TEXT,\n    crawled_at TEXT\n  );\n\n  CREATE VIRTUAL TABLE IF NOT EXISTS docs_fts USING fts5(\n    provider,\n    category,\n    title,\n    content,\n    content='docs',\n    content_rowid='id',\n    tokenize='porter unicode61'\n  );\n\n  CREATE TRIGGER IF NOT EXISTS docs_ai AFTER INSERT ON docs BEGIN\n    INSERT INTO docs_fts(rowid, provider, category, title, content)\n    VALUES (new.id, new.provider, new.category, new.title, new.content);\n  END;\n";
 export {};