@ontosdk/next 1.1.0 → 1.3.0

package/src/config.ts

@@ -0,0 +1,151 @@
+/**
+ * Configuration schema for onto.config.ts
+ * Used to dynamically generate llms.txt and other AI discovery files
+ */
+
+export type PageType = 'scoring' | 'about' | 'default';
+
+export interface OntoRoute {
+  /**
+   * The URL path (e.g., '/docs', '/api/reference')
+   */
+  path: string;
+  /**
+   * Description of what this route contains
+   */
+  description: string;
+  /**
+   * Optional: Page type for automatic JSON-LD schema injection
+   * - 'scoring': Injects Methodology schema with AIO scoring weights (40/35/25)
+   * - 'about': Injects Organization/AboutPage schema
+   * - 'default': No automatic schema injection
+   */
+  pageType?: PageType;
+}
+
+export interface OntoConfig {
+  /**
+   * The name of your project or site (required)
+   * Used as the H1 heading in llms.txt
+   */
+  name: string;
+
+  /**
+   * A short summary of your project (required)
+   * Displayed as a blockquote in llms.txt
+   * Should contain key information necessary for understanding the rest of the file
+   */
+  summary: string;
+
+  /**
+   * The base URL of your site (e.g., 'https://example.com')
+   */
+  baseUrl: string;
+
+  /**
+   * Optional: Additional sections to include in llms.txt
+   * Each section can contain any markdown content
+   */
+  sections?: {
+    heading: string;
+    content: string;
+  }[];
+
+  /**
+   * Key routes that AI agents should know about
+   * These will be formatted as a markdown list in llms.txt
+   */
+  routes?: OntoRoute[];
+
+  /**
+   * Optional: Links to external resources (documentation, API references, etc.)
+   */
+  externalLinks?: {
+    title: string;
+    url: string;
+    description?: string;
+  }[];
+
+  /**
+   * Optional: Organization information for JSON-LD schemas
+   */
+  organization?: {
+    name: string;
+    description?: string;
+    url?: string;
+    logo?: string;
+    foundingDate?: string;
+  };
+}
+
+/**
+ * Load the onto.config.ts file from the user's project
+ * This is used by the middleware to dynamically generate llms.txt
+ */
+export async function loadOntoConfig(): Promise<OntoConfig | null> {
+  try {
+    // Try to dynamically import the config file from the user's project root
+    // This runs in the middleware context, so we look in the project root
+    const config = await import(process.cwd() + '/onto.config');
+    return config.default || config;
+  } catch (error) {
+    // Config file doesn't exist or failed to load
+    return null;
+  }
+}
+
+/**
+ * Generate llms.txt content from OntoConfig
+ * Follows the llms.txt specification:
+ * - H1 with project name
+ * - Blockquote with summary
+ * - Additional markdown sections
+ */
+export function generateLlmsTxt(config: OntoConfig): string {
+  const lines: string[] = [];
+
+  // H1: Project name (required)
+  lines.push(`# ${config.name}`);
+  lines.push('');
+
+  // Blockquote: Summary (required)
+  lines.push(`> ${config.summary}`);
+  lines.push('');
+
+  // Key Routes section (if provided)
+  if (config.routes && config.routes.length > 0) {
+    lines.push('## Key Routes');
+    lines.push('');
+    for (const route of config.routes) {
+      const fullUrl = `${config.baseUrl}${route.path}`;
+      lines.push(`- [${route.path}](${fullUrl}): ${route.description}`);
+    }
+    lines.push('');
+  }
+
+  // External Links section (if provided)
+  if (config.externalLinks && config.externalLinks.length > 0) {
+    lines.push('## Resources');
+    lines.push('');
+    for (const link of config.externalLinks) {
+      if (link.description) {
+        lines.push(`- [${link.title}](${link.url}): ${link.description}`);
+      } else {
+        lines.push(`- [${link.title}](${link.url})`);
+      }
+    }
+    lines.push('');
+  }
+
+  // Custom sections (if provided)
+  if (config.sections && config.sections.length > 0) {
+    for (const section of config.sections) {
+      lines.push(`## ${section.heading}`);
+      lines.push('');
+      lines.push(section.content);
+      lines.push('');
+    }
+  }
+
+  return lines.join('\n').trim() + '\n';
+}

package/src/index.ts

@@ -1,3 +1,17 @@
 // We cannot use Webpack plugins reliably in Next.js Turbopack due to WorkerError restrictions.
 // Users must instead run `npx onto-next` as a postbuild script.
 export { extractContent } from './extractor';
+export { OntoConfig, OntoRoute, loadOntoConfig, generateLlmsTxt } from './config';
+export type { OntoConfig as OntoConfigType, OntoRoute as OntoRouteType, PageType } from './config';
+export {
+  generateAIOMethodologySchema,
+  generateOrganizationSchema,
+  generateAboutPageSchema,
+  generateSchemaForPageType,
+  serializeSchema
+} from './schemas';
+export type {
+  AIOMethodologySchema,
+  OrganizationSchema,
+  AboutPageSchema
+} from './schemas';

package/src/middleware.ts

@@ -1,29 +1,71 @@
 import { NextRequest, NextResponse } from 'next/server';
-
-const AI_BOT_USER_AGENTS = [
-    'GPTBot',
-    'ChatGPT-User',
-    'ClaudeBot',
-    'Claude-Web',
-    'anthropic-ai',
-    'PerplexityBot',
-    'OAI-SearchBot',
-    'GoogleExtended',
-];
+import { AI_BOT_USER_AGENTS, matchBot } from './bots';
+import { loadOntoConfig, generateLlmsTxt } from './config';
 
 export async function ontoMiddleware(request: NextRequest) {
-    const userAgent = request.headers.get('user-agent') || '';
+    const userAgent = request.headers.get('user-agent');
     const accept = request.headers.get('accept') || '';
 
-    const isAiBot = AI_BOT_USER_AGENTS.some(bot => userAgent.includes(bot));
+    const matched = matchBot(userAgent);
+    const isAiBot = !!matched;
     const isMarkdownRequested = accept.includes('text/markdown');
 
-    // If traffic is identified as an AI Bot, rewrite the URL
+    // If traffic is identified as an AI Bot or markdown is requested
     if (isAiBot || isMarkdownRequested) {
         const url = request.nextUrl.clone();
 
-        // Ignore internal next.js requests & static assets
-        if (url.pathname.startsWith('/_next') || url.pathname.includes('.')) {
+        // Ignore internal next.js requests & static assets (but not llms.txt)
+        if (url.pathname.startsWith('/_next')) {
+            return NextResponse.next();
+        }
+
+        // --- llms.txt Auto-Discovery ---
+        // Dynamically generate llms.txt from onto.config.ts
+        if (url.pathname === '/llms.txt') {
+            try {
+                const config = await loadOntoConfig();
+
+                if (config) {
+                    // Generate llms.txt dynamically from config
+                    const llmsTxtContent = generateLlmsTxt(config);
+                    const response = new NextResponse(llmsTxtContent, {
+                        headers: {
+                            'Content-Type': 'text/plain; charset=utf-8',
+                            'Cache-Control': 'public, max-age=3600, s-maxage=3600, stale-while-revalidate=86400',
+                        }
+                    });
+
+                    if (matched) {
+                        response.headers.set('X-Onto-Bot', `${matched.name} (${matched.company})`);
+                    }
+
+                    return response;
+                } else {
+                    // Fallback: try to serve static llms.txt from public folder
+                    url.pathname = '/llms.txt';
+                    const response = NextResponse.rewrite(url);
+                    response.headers.set('Content-Type', 'text/plain; charset=utf-8');
+                    response.headers.set('Cache-Control', 'public, max-age=3600, s-maxage=3600, stale-while-revalidate=86400');
+                    if (matched) {
+                        response.headers.set('X-Onto-Bot', `${matched.name} (${matched.company})`);
+                    }
+                    return response;
+                }
+            } catch (error) {
+                console.error('[Onto] Failed to generate llms.txt:', error);
+                // Fallback to static file on error
+                url.pathname = '/llms.txt';
+                const response = NextResponse.rewrite(url);
+                response.headers.set('Content-Type', 'text/plain; charset=utf-8');
+                if (matched) {
+                    response.headers.set('X-Onto-Bot', `${matched.name} (${matched.company})`);
+                }
+                return response;
+            }
+        }
+
+        // Skip other static assets
+        if (url.pathname.includes('.')) {
             return NextResponse.next();
         }
 
@@ -38,13 +80,21 @@ export async function ontoMiddleware(request: NextRequest) {
             payloadPath = payloadPath.slice(0, -1);
         }
 
+        // Common response headers for all bot responses
+        const botHeaders: Record<string, string> = {
+            'Content-Type': 'text/markdown; charset=utf-8',
+            'Cache-Control': 'public, max-age=3600, s-maxage=3600, stale-while-revalidate=86400',
+        };
+        if (matched) {
+            botHeaders['X-Onto-Bot'] = `${matched.name} (${matched.company})`;
+        }
+
         // --- Onto Control Plane Integration (Premium) ---
         const ONTO_API_KEY = process.env.ONTO_API_KEY;
         const DASHBOARD_URL = process.env.ONTO_DASHBOARD_URL || 'https://app.buildonto.dev';
 
         if (ONTO_API_KEY) {
-            // 1. Fire-and-forget tracking
-            // Use background fetch (no await) to avoid blocking the response
+            // 1. Fire-and-forget tracking — includes structured bot info
             fetch(`${DASHBOARD_URL}/api/track`, {
                 method: 'POST',
                 headers: {
@@ -54,15 +104,15 @@ export async function ontoMiddleware(request: NextRequest) {
                 body: JSON.stringify({
                     route: url.pathname,
                     userAgent: userAgent,
+                    bot: matched ? matched.name : null,
+                    company: matched ? matched.company : null,
                 })
             }).catch(() => {});
 
             // 2. Dynamic Context Injection
             try {
-                // Fetch the injection from the Control Plane
                 const injectRes = await fetch(`${DASHBOARD_URL}/api/sdk/inject?route=${url.pathname}`, {
                     headers: { 'x-onto-key': ONTO_API_KEY },
-                    // Set a strict timeout to keep edge fast
                     signal: AbortSignal.timeout(1500)
                 });
 
@@ -70,7 +120,6 @@ export async function ontoMiddleware(request: NextRequest) {
                     const { injection } = await injectRes.json();
                     
                     if (injection) {
-                        // To inject, we must fetch the local markdown and append
                         const localMdUrl = `${url.origin}/.onto${payloadPath}.md`;
                         const mdRes = await fetch(localMdUrl);
                         
@@ -80,8 +129,7 @@ export async function ontoMiddleware(request: NextRequest) {
                             
                             return new NextResponse(finalMarkdown, {
                                 headers: {
-                                    'Content-Type': 'text/markdown; charset=utf-8',
-                                    'Cache-Control': 'public, max-age=3600, s-maxage=3600, stale-while-revalidate=86400',
+                                    ...botHeaders,
                                     'X-Onto-Injected': 'true'
                                 }
                             });
@@ -97,8 +145,18 @@ export async function ontoMiddleware(request: NextRequest) {
         url.pathname = `/.onto${payloadPath}.md`;
 
         // Rewrite implicitly serves the target URL transparently to the client.
-        return NextResponse.rewrite(url);
+        const response = NextResponse.rewrite(url);
+        if (matched) {
+            response.headers.set('X-Onto-Bot', `${matched.name} (${matched.company})`);
+        }
+        return response;
     }
 
     return NextResponse.next();
 }
+
+// Re-export the bot registry for consumers who want to extend or inspect it
+export { AI_BOT_USER_AGENTS, matchBot } from './bots';
+export type { AiBot } from './bots';
+
+

package/src/schemas.ts

@@ -0,0 +1,186 @@
+/**
+ * JSON-LD Schema generators for automatic structured data injection
+ * Follows Schema.org standards for AI-friendly metadata
+ */
+
+import { OntoConfig } from './config';
+
+/**
+ * Standard AIO (AI Optimization) scoring methodology
+ * Based on the Onto scoring algorithm:
+ * - Content Negotiation: 40% (30 points)
+ * - React Tax / Token Efficiency: 35% (30 points)
+ * - Structured Data: 25% (25 points)
+ * - Semantic HTML: Bonus (15 points)
+ */
+export interface AIOMethodologySchema {
+  '@context': 'https://schema.org';
+  '@type': 'HowTo';
+  name: string;
+  description: string;
+  step: Array<{
+    '@type': 'HowToStep';
+    name: string;
+    text: string;
+    position: number;
+  }>;
+}
+
+/**
+ * Generate AIO Scoring Methodology JSON-LD schema
+ * This explains to AI agents how the scoring system works
+ */
+export function generateAIOMethodologySchema(
+  config: OntoConfig,
+  pageUrl: string
+): AIOMethodologySchema {
+  return {
+    '@context': 'https://schema.org',
+    '@type': 'HowTo',
+    name: 'AIO Score Calculation Methodology',
+    description: 'AI Optimization (AIO) Score measures how well a website is optimized for AI agents and LLM crawlers. Scored out of 100 points based on four key metrics.',
+    step: [
+      {
+        '@type': 'HowToStep',
+        name: 'Content Negotiation',
+        text: 'Check if the site responds to Accept: text/markdown header. Weight: 40%. Penalty: -30 points if missing. This ensures AI bots receive optimized content instead of heavy HTML.',
+        position: 1
+      },
+      {
+        '@type': 'HowToStep',
+        name: 'Token Efficiency (React Tax)',
+        text: 'Measure the ratio of visible text to total HTML size. Weight: 35%. Penalty: -30 points if HTML > 100KB but text < 1KB. Detects JavaScript-heavy sites that are difficult for AI to parse.',
+        position: 2
+      },
+      {
+        '@type': 'HowToStep',
+        name: 'Structured Data',
+        text: 'Verify presence of JSON-LD structured data (Schema.org). Weight: 25%. Penalty: -25 points if missing. Enables AI to confidently extract pricing, products, and entities.',
+        position: 3
+      },
+      {
+        '@type': 'HowToStep',
+        name: 'Semantic HTML',
+        text: 'Check for semantic tags like <main> and <article>. Bonus: +15 points if present. Helps AI agents separate navigation from core content.',
+        position: 4
+      }
+    ]
+  };
+}
+
+/**
+ * Organization schema for About pages
+ */
+export interface OrganizationSchema {
+  '@context': 'https://schema.org';
+  '@type': 'Organization';
+  name: string;
+  url?: string;
+  description?: string;
+  logo?: string;
+  foundingDate?: string;
+}
+
+/**
+ * Generate Organization JSON-LD schema for About pages
+ */
+export function generateOrganizationSchema(
+  config: OntoConfig,
+  pageUrl: string
+): OrganizationSchema | null {
+  if (!config.organization) {
+    return null;
+  }
+
+  const schema: OrganizationSchema = {
+    '@context': 'https://schema.org',
+    '@type': 'Organization',
+    name: config.organization.name
+  };
+
+  if (config.organization.url) {
+    schema.url = config.organization.url;
+  }
+
+  if (config.organization.description) {
+    schema.description = config.organization.description;
+  }
+
+  if (config.organization.logo) {
+    schema.logo = config.organization.logo;
+  }
+
+  if (config.organization.foundingDate) {
+    schema.foundingDate = config.organization.foundingDate;
+  }
+
+  return schema;
+}
+
+/**
+ * AboutPage schema combining Organization and WebPage
+ */
+export interface AboutPageSchema {
+  '@context': 'https://schema.org';
+  '@type': 'AboutPage';
+  name: string;
+  url: string;
+  description?: string;
+  mainEntity?: OrganizationSchema;
+}
+
+/**
+ * Generate AboutPage JSON-LD schema
+ */
+export function generateAboutPageSchema(
+  config: OntoConfig,
+  pageUrl: string
+): AboutPageSchema {
+  const orgSchema = generateOrganizationSchema(config, pageUrl);
+
+  const schema: AboutPageSchema = {
+    '@context': 'https://schema.org',
+    '@type': 'AboutPage',
+    name: `About ${config.name}`,
+    url: pageUrl
+  };
+
+  if (config.summary) {
+    schema.description = config.summary;
+  }
+
+  if (orgSchema) {
+    schema.mainEntity = orgSchema;
+  }
+
+  return schema;
+}
+
+/**
+ * Determine which schema to generate based on page type
+ */
+export function generateSchemaForPageType(
+  pageType: 'scoring' | 'about' | 'default',
+  config: OntoConfig,
+  pageUrl: string
+): any | null {
+  switch (pageType) {
+    case 'scoring':
+      return generateAIOMethodologySchema(config, pageUrl);
+    case 'about':
+      return generateAboutPageSchema(config, pageUrl);
+    case 'default':
+    default:
+      return null;
+  }
+}
+
+/**
+ * Serialize schema to JSON-LD script tag content
+ */
+export function serializeSchema(schema: any | null): string | null {
+  if (!schema) {
+    return null;
+  }
+  return JSON.stringify(schema, null, 2);
+}

package/tsconfig.json

@@ -7,6 +7,7 @@
         "strict": true,
         "skipLibCheck": true,
         "forceConsistentCasingInFileNames": true,
+        "jsx": "react-jsx",
         "outDir": "dist"
     },
     "include": [

package/tsup.config.ts

@@ -1,7 +1,7 @@
 import { defineConfig } from 'tsup';
 
 export default defineConfig({
-    entry: ['src/index.ts', 'src/cli.ts', 'src/middleware.ts'],
+    entry: ['src/index.ts', 'src/cli.ts', 'src/middleware.ts', 'src/OntoHead.tsx', 'src/OntoProvider.tsx', 'src/config.ts', 'src/schemas.ts'],
     format: ['cjs', 'esm'],
     dts: true,
     splitting: false,
@@ -10,5 +10,5 @@ export default defineConfig({
     bundle: true,
     outDir: 'dist',
     minify: true,
-    external: ['next'],
+    external: ['next', 'react'],
 });