@ontosdk/next 1.0.0

package/dist/cli.d.mts

@@ -0,0 +1 @@
+#!/usr/bin/env node

package/dist/cli.d.ts

@@ -0,0 +1 @@
+#!/usr/bin/env node

package/dist/cli.js

@@ -0,0 +1,13 @@
+#!/usr/bin/env node
+"use strict";var j=Object.create;var $=Object.defineProperty;var v=Object.getOwnPropertyDescriptor;var z=Object.getOwnPropertyNames;var P=Object.getPrototypeOf,R=Object.prototype.hasOwnProperty;var N=(t,n,o,e)=>{if(n&&typeof n=="object"||typeof n=="function")for(let r of z(n))!R.call(t,r)&&r!==o&&$(t,r,{get:()=>n[r],enumerable:!(e=v(n,r))||e.enumerable});return t};var h=(t,n,o)=>(o=t!=null?j(P(t)):{},N(n||!t||!t.__esModule?$(o,"default",{value:t,enumerable:!0}):o,t));var b=require("glob"),c=h(require("fs")),p=h(require("path")),i=h(require("picocolors"));var k=h(require("cheerio")),O=h(require("turndown")),E=new O.default({headingStyle:"atx",codeBlockStyle:"fenced"});function F(t,n="Generated Output"){let o=t.length,e=k.load(t),r=e("title").text()||e("h1").first().text()||"Untitled Page",f=e('meta[name="description"]').attr("content")||"No description found.",l=[];e('script[type="application/ld+json"]').each((g,w)=>{try{let x=e(w).html()||"",m=JSON.parse(x);l.push(m)}catch{}}),e("script, style, noscript, iframe, svg, nav, footer, meta, link, header").remove();let s="";e("main").length>0?s=e("main").html()||"":e("article").length>0?s=e("article").html()||"":s=e("body").html()||"";let S=E.turndown(s),a=[`# ${r}`,`> ${f}`,"",`**Source:** ${n}`,`**Extracted:** ${new Date().toISOString()}`,"","---",""].join(`
+`)+S;l.length>0&&(a+=`
+
+---
+## Structured Data (JSON-LD)
+\`\`\`json
+`,l.forEach(g=>{a+=JSON.stringify(g,null,2)+`
+`}),a+="```\n");let u=a.length,d=o>0?(o-u)/o*100:0;return{markdown:a,metadata:{title:r,description:f,jsonLd:l},stats:{originalHtmlSize:o,markdownSize:u,tokenReductionRatio:d}}}async function L(){console.log(i.default.cyan(`
+[Onto] Starting Semantic Output Generation...`));let t=process.cwd(),n=p.default.join(t,".next/server/app"),o=p.default.join(t,"public/.onto");if(!c.default.existsSync(n)){console.log(i.default.yellow(`[Onto] Could not find Next.js app output at ${n}`)),console.log(i.default.yellow('[Onto] Ensure this is run after "next build" and you are using the App Router.'));return}let e=await(0,b.glob)("**/*.html",{cwd:n});if(e.length===0){console.log(i.default.yellow("[Onto] No static HTML files found to process."));return}c.default.existsSync(o)||c.default.mkdirSync(o,{recursive:!0});let r=0,f=0,l=0;for(let s of e){let S=p.default.join(n,s),y=s.replace(/\.html$/,".md"),a=p.default.join(o,y);try{let u=c.default.readFileSync(S,"utf8"),d=F(u,`/${y.replace(/\.md$/,"")}`),g=p.default.dirname(a);c.default.existsSync(g)||c.default.mkdirSync(g,{recursive:!0}),c.default.writeFileSync(a,d.markdown,"utf8"),r+=d.stats.originalHtmlSize,f+=d.stats.markdownSize,l++;let w=(d.stats.originalHtmlSize/1024).toFixed(1),x=(d.stats.markdownSize/1024).toFixed(1),m=s.replace(/\.html$/,"");m==="index"?m="/":m=`/${m}`,console.log(i.default.green("\u2713 Optimized")+i.default.dim(` ${m} `)+i.default.blue(`[${w}KB -> ${x}KB]`))}catch(u){console.error(i.default.red(`\u2717 Failed to process ${s}: ${u.message}`))}}console.log(i.default.cyan(`
+[Onto] Finished generation.`)),console.log(i.default.bold(i.default.magenta(`Processed ${l} pages. Total Size: ${(r/1024).toFixed(1)}KB -> ${(f/1024).toFixed(1)}KB`))),console.log(i.default.dim(`Edge payloads are ready at /public/.onto/*
+`))}L().catch(t=>{console.error(i.default.red(`[Onto] Fatal Error: ${t.message}`)),process.exit(1)});
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/cli.ts","../src/extractor.ts"],"sourcesContent":["#!/usr/bin/env node\r\nimport { glob } from 'glob';\r\nimport fs from 'fs';\r\nimport path from 'path';\r\nimport pc from 'picocolors';\r\nimport { extractContent } from './extractor';\r\n\r\nasync function main() {\r\n    console.log(pc.cyan('\\n[Onto] Starting Semantic Output Generation...'));\r\n\r\n    const cwd = process.cwd();\r\n    const nextAppDirDir = path.join(cwd, '.next/server/app');\r\n    const ontoPublicDir = path.join(cwd, 'public/.onto');\r\n\r\n    if (!fs.existsSync(nextAppDirDir)) {\r\n        console.log(pc.yellow(`[Onto] Could not find Next.js app output at ${nextAppDirDir}`));\r\n        console.log(pc.yellow(`[Onto] Ensure this is run after \"next build\" and you are using the App Router.`));\r\n        return;\r\n    }\r\n\r\n    // Find all HTML files rendered by Next.js in the app directory\r\n    const files = await glob('**/*.html', { cwd: nextAppDirDir });\r\n\r\n    if (files.length === 0) {\r\n        console.log(pc.yellow(`[Onto] No static HTML files found to process.`));\r\n        return;\r\n    }\r\n\r\n    // Ensure output directory exists\r\n    if (!fs.existsSync(ontoPublicDir)) {\r\n        fs.mkdirSync(ontoPublicDir, { recursive: true });\r\n    }\r\n\r\n    let totalOriginalSize = 0;\r\n    let totalMarkdownSize = 0;\r\n    let totalFilesProcessed = 0;\r\n\r\n    for (const file of files) {\r\n        const inputPath = path.join(nextAppDirDir, file);\r\n\r\n        // We map file path e.g. \"pricing.html\" to \"pricing.md\", or \"blog/post.html\" to \"blog/post.md\"\r\n        let outputPathRelative = file.replace(/\\.html$/, '.md');\r\n        // If it's a dynamic route page, or purely root index.html\r\n        const outputPath = path.join(ontoPublicDir, outputPathRelative);\r\n\r\n        try {\r\n            const htmlContent = fs.readFileSync(inputPath, 'utf8');\r\n\r\n            const result = extractContent(htmlContent, `/${outputPathRelative.replace(/\\.md$/, '')}`);\r\n\r\n            // Ensure specific sub-directory exists (e.g., for blog/post.md)\r\n            const outputDir = path.dirname(outputPath);\r\n            if (!fs.existsSync(outputDir)) {\r\n                fs.mkdirSync(outputDir, { recursive: true });\r\n            }\r\n\r\n            fs.writeFileSync(outputPath, result.markdown, 'utf8');\r\n\r\n            totalOriginalSize += result.stats.originalHtmlSize;\r\n            totalMarkdownSize += result.stats.markdownSize;\r\n            totalFilesProcessed++;\r\n\r\n            const origKb = (result.stats.originalHtmlSize / 1024).toFixed(1);\r\n            const mdKb = (result.stats.markdownSize / 1024).toFixed(1);\r\n\r\n            // /index.html -> /\r\n            let routeName = file.replace(/\\.html$/, '');\r\n            if (routeName === 'index') routeName = '/';\r\n            else routeName = `/${routeName}`;\r\n\r\n            console.log(\r\n                pc.green(`✓ Optimized`) +\r\n                pc.dim(` ${routeName} `) +\r\n                pc.blue(`[${origKb}KB -> ${mdKb}KB]`)\r\n            );\r\n        } catch (e: any) {\r\n            console.error(pc.red(`✗ Failed to process ${file}: ${e.message}`));\r\n        }\r\n    }\r\n\r\n    console.log(pc.cyan(`\\n[Onto] Finished generation.`));\r\n    console.log(\r\n        pc.bold(\r\n            pc.magenta(`Processed ${totalFilesProcessed} pages. Total Size: ${(totalOriginalSize / 1024).toFixed(1)}KB -> ${(totalMarkdownSize / 1024).toFixed(1)}KB`)\r\n        )\r\n    );\r\n    console.log(pc.dim(`Edge payloads are ready at /public/.onto/*\\n`));\r\n}\r\n\r\nmain().catch(e => {\r\n    console.error(pc.red(`[Onto] Fatal Error: ${e.message}`));\r\n    process.exit(1);\r\n});\r\n","import * as cheerio from 'cheerio';\r\nimport TurndownService from 'turndown';\r\n\r\nconst turndownService = new TurndownService({\r\n    headingStyle: 'atx',\r\n    codeBlockStyle: 'fenced',\r\n});\r\n\r\n// Configure turndown to keep some layout or handle semantic tags differently if needed\r\n\r\nexport interface ExtractionResult {\r\n    markdown: string;\r\n    metadata: {\r\n        title: string;\r\n        description: string;\r\n        jsonLd: any[];\r\n    };\r\n    stats: {\r\n        originalHtmlSize: number;\r\n        markdownSize: number;\r\n        tokenReductionRatio: number;\r\n    };\r\n}\r\n\r\n/**\r\n * Extracts pure semantic markdown and metadata from rendered Next.js HTML strings.\r\n * @param html The raw HTML string.\r\n * @param sourceUrl (Optional) the URL this was generated from, to attach as metadata.\r\n * @returns {ExtractionResult} The extracted payload.\r\n */\r\nexport function extractContent(html: string, sourceUrl: string = 'Generated Output'): ExtractionResult {\r\n    const originalSize = html.length;\r\n\r\n    const $ = cheerio.load(html);\r\n\r\n    // 1. Extract Metadata BEFORE removing structure\r\n    const title = $('title').text() || $('h1').first().text() || 'Untitled Page';\r\n    const description = $('meta[name=\"description\"]').attr('content') || 'No description found.';\r\n\r\n    const jsonLdScripts: any[] = [];\r\n    $('script[type=\"application/ld+json\"]').each((_, el) => {\r\n        try {\r\n            const raw = $(el).html() || '';\r\n            const parsed = JSON.parse(raw);\r\n            jsonLdScripts.push(parsed);\r\n        } catch {\r\n            // ignore bad json\r\n        }\r\n    });\r\n\r\n    // 2. Strip noise (React boilerplate, styles, unnecessary tags)\r\n    $('script, style, noscript, iframe, svg, nav, footer, meta, link, header').remove();\r\n\r\n    // Optionally remove typical Next.js hidden wrappers if they don't contain real content.\r\n    // Next.js uses <div id=\"__next\"> but we mostly just want semantic content.\r\n\r\n    // 3. Find the entry point for content\r\n    // Prefer <main> or <article> over <body>\r\n    let contentHtml = '';\r\n    if ($('main').length > 0) {\r\n        contentHtml = $('main').html() || '';\r\n    } else if ($('article').length > 0) {\r\n        contentHtml = $('article').html() || '';\r\n    } else {\r\n        contentHtml = $('body').html() || '';\r\n    }\r\n\r\n    // 4. Convert to Markdown\r\n    let markdown = turndownService.turndown(contentHtml);\r\n\r\n    // 5. Optionally inject Metadata header\r\n    const headerLines = [\r\n        `# ${title}`,\r\n        `> ${description}`,\r\n        ``,\r\n        `**Source:** ${sourceUrl}`,\r\n        `**Extracted:** ${new Date().toISOString()}`,\r\n        ``,\r\n        `---`,\r\n        ``\r\n    ];\r\n\r\n    let finalMarkdown = headerLines.join('\\n') + markdown;\r\n\r\n    // Add JSON-LD section if exists\r\n    if (jsonLdScripts.length > 0) {\r\n        finalMarkdown += '\\n\\n---\\n## Structured Data (JSON-LD)\\n```json\\n';\r\n        jsonLdScripts.forEach(j => {\r\n            finalMarkdown += JSON.stringify(j, null, 2) + '\\n';\r\n        });\r\n        finalMarkdown += '```\\n';\r\n    }\r\n\r\n    const markdownSize = finalMarkdown.length;\r\n    const tokenReductionRatio = originalSize > 0 ? ((originalSize - markdownSize) / originalSize) * 100 : 0;\r\n\r\n    return {\r\n        markdown: finalMarkdown,\r\n        metadata: {\r\n            title,\r\n            description,\r\n            jsonLd: jsonLdScripts\r\n        },\r\n        stats: {\r\n            originalHtmlSize: originalSize,\r\n            markdownSize,\r\n            tokenReductionRatio\r\n        }\r\n    };\r\n}\r\n\r\nexport async function generateStaticPayloads(nextAppDirDir: string, ontoPublicDir: string) {\r\n    const fs = await import('fs');\r\n    const path = await import('path');\r\n    const { glob } = await import('glob');\r\n\r\n    if (!fs.existsSync(nextAppDirDir)) {\r\n        return;\r\n    }\r\n\r\n    const files = await glob('**/*.html', { cwd: nextAppDirDir });\r\n    if (files.length === 0) return;\r\n\r\n    if (!fs.existsSync(ontoPublicDir)) {\r\n        fs.mkdirSync(ontoPublicDir, { recursive: true });\r\n    }\r\n\r\n    let totalFilesProcessed = 0;\r\n\r\n    for (const file of files) {\r\n        const inputPath = path.join(nextAppDirDir, file);\r\n        const outputPathRelative = file.replace(/\\.html$/, '.md');\r\n        const outputPath = path.join(ontoPublicDir, outputPathRelative);\r\n\r\n        try {\r\n            const htmlContent = fs.readFileSync(inputPath, 'utf8');\r\n\r\n            let routeName = file.replace(/\\.html$/, '');\r\n            if (routeName === 'index') routeName = '/';\r\n            else routeName = `/${routeName}`;\r\n\r\n            const result = extractContent(htmlContent, routeName);\r\n\r\n            const outputDir = path.dirname(outputPath);\r\n            if (!fs.existsSync(outputDir)) {\r\n                fs.mkdirSync(outputDir, { recursive: true });\r\n            }\r\n\r\n            fs.writeFileSync(outputPath, result.markdown, 'utf8');\r\n            totalFilesProcessed++;\r\n        } catch (e: any) {\r\n            console.error(`[Onto] Failed to process ${file}: ${e.message}`);\r\n        }\r\n    }\r\n    console.log(`[Onto] Successfully generated ${totalFilesProcessed} semantic markdown endpoints.`);\r\n}\r\n"],"mappings":";wdACA,IAAAA,EAAqB,gBACrBC,EAAe,iBACfC,EAAiB,mBACjBC,EAAe,yBCJf,IAAAC,EAAyB,sBACzBC,EAA4B,uBAEtBC,EAAkB,IAAI,EAAAC,QAAgB,CACxC,aAAc,MACd,eAAgB,QACpB,CAAC,EAwBM,SAASC,EAAeC,EAAcC,EAAoB,mBAAsC,CACnG,IAAMC,EAAeF,EAAK,OAEpBG,EAAY,OAAKH,CAAI,EAGrBI,EAAQD,EAAE,OAAO,EAAE,KAAK,GAAKA,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,GAAK,gBACvDE,EAAcF,EAAE,0BAA0B,EAAE,KAAK,SAAS,GAAK,wBAE/DG,EAAuB,CAAC,EAC9BH,EAAE,oCAAoC,EAAE,KAAK,CAACI,EAAGC,IAAO,CACpD,GAAI,CACA,IAAMC,EAAMN,EAAEK,CAAE,EAAE,KAAK,GAAK,GACtBE,EAAS,KAAK,MAAMD,CAAG,EAC7BH,EAAc,KAAKI,CAAM,CAC7B,MAAQ,CAER,CACJ,CAAC,EAGDP,EAAE,uEAAuE,EAAE,OAAO,EAOlF,IAAIQ,EAAc,GACdR,EAAE,MAAM,EAAE,OAAS,EACnBQ,EAAcR,EAAE,MAAM,EAAE,KAAK,GAAK,GAC3BA,EAAE,SAAS,EAAE,OAAS,EAC7BQ,EAAcR,EAAE,SAAS,EAAE,KAAK,GAAK,GAErCQ,EAAcR,EAAE,MAAM,EAAE,KAAK,GAAK,GAItC,IAAIS,EAAWf,EAAgB,SAASc,CAAW,EAc/CE,EAXgB,CAChB,KAAKT,CAAK,GACV,KAAKC,CAAW,GAChB,GACA,eAAeJ,CAAS,GACxB,kBAAkB,IAAI,KAAK,EAAE,YAAY,CAAC,GAC1C,GACA,MACA,EACJ,EAEgC,KAAK;AAAA,CAAI,EAAIW,EAGzCN,EAAc,OAAS,IACvBO,GAAiB;AAAA;AAAA;AAAA;AAAA;AAAA,EACjBP,EAAc,QAAQQ,GAAK,CACvBD,GAAiB,KAAK,UAAUC,EAAG,KAAM,CAAC,EAAI;AAAA,CAClD,CAAC,EACDD,GAAiB,SAGrB,IAAME,EAAeF,EAAc,OAC7BG,EAAsBd,EAAe,GAAMA,EAAea,GAAgBb,EAAgB,IAAM,EAEtG,MAAO,CACH,SAAUW,EACV,SAAU,CACN,MAAAT,EACA,YAAAC,EACA,OAAQC,CACZ,EACA,MAAO,CACH,iBAAkBJ,EAClB,aAAAa,EACA,oBAAAC,CACJ,CACJ,CACJ,CDtGA,eAAeC,GAAO,CAClB,QAAQ,IAAI,EAAAC,QAAG,KAAK;AAAA,8CAAiD,CAAC,EAEtE,IAAMC,EAAM,QAAQ,IAAI,EAClBC,EAAgB,EAAAC,QAAK,KAAKF,EAAK,kBAAkB,EACjDG,EAAgB,EAAAD,QAAK,KAAKF,EAAK,cAAc,EAEnD,GAAI,CAAC,EAAAI,QAAG,WAAWH,CAAa,EAAG,CAC/B,QAAQ,IAAI,EAAAF,QAAG,OAAO,+CAA+CE,CAAa,EAAE,CAAC,EACrF,QAAQ,IAAI,EAAAF,QAAG,OAAO,gFAAgF,CAAC,EACvG,MACJ,CAGA,IAAMM,EAAQ,QAAM,QAAK,YAAa,CAAE,IAAKJ,CAAc,CAAC,EAE5D,GAAII,EAAM,SAAW,EAAG,CACpB,QAAQ,IAAI,EAAAN,QAAG,OAAO,+CAA+C,CAAC,EACtE,MACJ,CAGK,EAAAK,QAAG,WAAWD,CAAa,GAC5B,EAAAC,QAAG,UAAUD,EAAe,CAAE,UAAW,EAAK,CAAC,EAGnD,IAAIG,EAAoB,EACpBC,EAAoB,EACpBC,EAAsB,EAE1B,QAAWC,KAAQJ,EAAO,CACtB,IAAMK,EAAY,EAAAR,QAAK,KAAKD,EAAeQ,CAAI,EAG3CE,EAAqBF,EAAK,QAAQ,UAAW,KAAK,EAEhDG,EAAa,EAAAV,QAAK,KAAKC,EAAeQ,CAAkB,EAE9D,GAAI,CACA,IAAME,EAAc,EAAAT,QAAG,aAAaM,EAAW,MAAM,EAE/CI,EAASC,EAAeF,EAAa,IAAIF,EAAmB,QAAQ,QAAS,EAAE,CAAC,EAAE,EAGlFK,EAAY,EAAAd,QAAK,QAAQU,CAAU,EACpC,EAAAR,QAAG,WAAWY,CAAS,GACxB,EAAAZ,QAAG,UAAUY,EAAW,CAAE,UAAW,EAAK,CAAC,EAG/C,EAAAZ,QAAG,cAAcQ,EAAYE,EAAO,SAAU,MAAM,EAEpDR,GAAqBQ,EAAO,MAAM,iBAClCP,GAAqBO,EAAO,MAAM,aAClCN,IAEA,IAAMS,GAAUH,EAAO,MAAM,iBAAmB,MAAM,QAAQ,CAAC,EACzDI,GAAQJ,EAAO,MAAM,aAAe,MAAM,QAAQ,CAAC,EAGrDK,EAAYV,EAAK,QAAQ,UAAW,EAAE,EACtCU,IAAc,QAASA,EAAY,IAClCA,EAAY,IAAIA,CAAS,GAE9B,QAAQ,IACJ,EAAApB,QAAG,MAAM,kBAAa,EACtB,EAAAA,QAAG,IAAI,IAAIoB,CAAS,GAAG,EACvB,EAAApB,QAAG,KAAK,IAAIkB,CAAM,SAASC,CAAI,KAAK,CACxC,CACJ,OAASE,EAAQ,CACb,QAAQ,MAAM,EAAArB,QAAG,IAAI,4BAAuBU,CAAI,KAAKW,EAAE,OAAO,EAAE,CAAC,CACrE,CACJ,CAEA,QAAQ,IAAI,EAAArB,QAAG,KAAK;AAAA,4BAA+B,CAAC,EACpD,QAAQ,IACJ,EAAAA,QAAG,KACC,EAAAA,QAAG,QAAQ,aAAaS,CAAmB,wBAAwBF,EAAoB,MAAM,QAAQ,CAAC,CAAC,UAAUC,EAAoB,MAAM,QAAQ,CAAC,CAAC,IAAI,CAC7J,CACJ,EACA,QAAQ,IAAI,EAAAR,QAAG,IAAI;AAAA,CAA8C,CAAC,CACtE,CAEAD,EAAK,EAAE,MAAMsB,GAAK,CACd,QAAQ,MAAM,EAAArB,QAAG,IAAI,uBAAuBqB,EAAE,OAAO,EAAE,CAAC,EACxD,QAAQ,KAAK,CAAC,CAClB,CAAC","names":["import_glob","import_fs","import_path","import_picocolors","cheerio","import_turndown","turndownService","TurndownService","extractContent","html","sourceUrl","originalSize","$","title","description","jsonLdScripts","_","el","raw","parsed","contentHtml","markdown","finalMarkdown","j","markdownSize","tokenReductionRatio","main","pc","cwd","nextAppDirDir","path","ontoPublicDir","fs","files","totalOriginalSize","totalMarkdownSize","totalFilesProcessed","file","inputPath","outputPathRelative","outputPath","htmlContent","result","extractContent","outputDir","origKb","mdKb","routeName","e"]}

package/dist/cli.mjs

@@ -0,0 +1,13 @@
+#!/usr/bin/env node
+import{glob as F}from"glob";import l from"fs";import g from"path";import e from"picocolors";import*as x from"cheerio";import k from"turndown";var O=new k({headingStyle:"atx",codeBlockStyle:"fenced"});function $(r,d="Generated Output"){let o=r.length,t=x.load(r),u=t("title").text()||t("h1").first().text()||"Untitled Page",p=t('meta[name="description"]').attr("content")||"No description found.",s=[];t('script[type="application/ld+json"]').each((f,y)=>{try{let w=t(y).html()||"",c=JSON.parse(w);s.push(c)}catch{}}),t("script, style, noscript, iframe, svg, nav, footer, meta, link, header").remove();let n="";t("main").length>0?n=t("main").html()||"":t("article").length>0?n=t("article").html()||"":n=t("body").html()||"";let h=O.turndown(n),i=[`# ${u}`,`> ${p}`,"",`**Source:** ${d}`,`**Extracted:** ${new Date().toISOString()}`,"","---",""].join(`
+`)+h;s.length>0&&(i+=`
+
+---
+## Structured Data (JSON-LD)
+\`\`\`json
+`,s.forEach(f=>{i+=JSON.stringify(f,null,2)+`
+`}),i+="```\n");let m=i.length,a=o>0?(o-m)/o*100:0;return{markdown:i,metadata:{title:u,description:p,jsonLd:s},stats:{originalHtmlSize:o,markdownSize:m,tokenReductionRatio:a}}}async function b(){console.log(e.cyan(`
+[Onto] Starting Semantic Output Generation...`));let r=process.cwd(),d=g.join(r,".next/server/app"),o=g.join(r,"public/.onto");if(!l.existsSync(d)){console.log(e.yellow(`[Onto] Could not find Next.js app output at ${d}`)),console.log(e.yellow('[Onto] Ensure this is run after "next build" and you are using the App Router.'));return}let t=await F("**/*.html",{cwd:d});if(t.length===0){console.log(e.yellow("[Onto] No static HTML files found to process."));return}l.existsSync(o)||l.mkdirSync(o,{recursive:!0});let u=0,p=0,s=0;for(let n of t){let h=g.join(d,n),S=n.replace(/\.html$/,".md"),i=g.join(o,S);try{let m=l.readFileSync(h,"utf8"),a=$(m,`/${S.replace(/\.md$/,"")}`),f=g.dirname(i);l.existsSync(f)||l.mkdirSync(f,{recursive:!0}),l.writeFileSync(i,a.markdown,"utf8"),u+=a.stats.originalHtmlSize,p+=a.stats.markdownSize,s++;let y=(a.stats.originalHtmlSize/1024).toFixed(1),w=(a.stats.markdownSize/1024).toFixed(1),c=n.replace(/\.html$/,"");c==="index"?c="/":c=`/${c}`,console.log(e.green("\u2713 Optimized")+e.dim(` ${c} `)+e.blue(`[${y}KB -> ${w}KB]`))}catch(m){console.error(e.red(`\u2717 Failed to process ${n}: ${m.message}`))}}console.log(e.cyan(`
+[Onto] Finished generation.`)),console.log(e.bold(e.magenta(`Processed ${s} pages. Total Size: ${(u/1024).toFixed(1)}KB -> ${(p/1024).toFixed(1)}KB`))),console.log(e.dim(`Edge payloads are ready at /public/.onto/*
+`))}b().catch(r=>{console.error(e.red(`[Onto] Fatal Error: ${r.message}`)),process.exit(1)});
+//# sourceMappingURL=cli.mjs.map

package/dist/cli.mjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/cli.ts","../src/extractor.ts"],"sourcesContent":["#!/usr/bin/env node\r\nimport { glob } from 'glob';\r\nimport fs from 'fs';\r\nimport path from 'path';\r\nimport pc from 'picocolors';\r\nimport { extractContent } from './extractor';\r\n\r\nasync function main() {\r\n    console.log(pc.cyan('\\n[Onto] Starting Semantic Output Generation...'));\r\n\r\n    const cwd = process.cwd();\r\n    const nextAppDirDir = path.join(cwd, '.next/server/app');\r\n    const ontoPublicDir = path.join(cwd, 'public/.onto');\r\n\r\n    if (!fs.existsSync(nextAppDirDir)) {\r\n        console.log(pc.yellow(`[Onto] Could not find Next.js app output at ${nextAppDirDir}`));\r\n        console.log(pc.yellow(`[Onto] Ensure this is run after \"next build\" and you are using the App Router.`));\r\n        return;\r\n    }\r\n\r\n    // Find all HTML files rendered by Next.js in the app directory\r\n    const files = await glob('**/*.html', { cwd: nextAppDirDir });\r\n\r\n    if (files.length === 0) {\r\n        console.log(pc.yellow(`[Onto] No static HTML files found to process.`));\r\n        return;\r\n    }\r\n\r\n    // Ensure output directory exists\r\n    if (!fs.existsSync(ontoPublicDir)) {\r\n        fs.mkdirSync(ontoPublicDir, { recursive: true });\r\n    }\r\n\r\n    let totalOriginalSize = 0;\r\n    let totalMarkdownSize = 0;\r\n    let totalFilesProcessed = 0;\r\n\r\n    for (const file of files) {\r\n        const inputPath = path.join(nextAppDirDir, file);\r\n\r\n        // We map file path e.g. \"pricing.html\" to \"pricing.md\", or \"blog/post.html\" to \"blog/post.md\"\r\n        let outputPathRelative = file.replace(/\\.html$/, '.md');\r\n        // If it's a dynamic route page, or purely root index.html\r\n        const outputPath = path.join(ontoPublicDir, outputPathRelative);\r\n\r\n        try {\r\n            const htmlContent = fs.readFileSync(inputPath, 'utf8');\r\n\r\n            const result = extractContent(htmlContent, `/${outputPathRelative.replace(/\\.md$/, '')}`);\r\n\r\n            // Ensure specific sub-directory exists (e.g., for blog/post.md)\r\n            const outputDir = path.dirname(outputPath);\r\n            if (!fs.existsSync(outputDir)) {\r\n                fs.mkdirSync(outputDir, { recursive: true });\r\n            }\r\n\r\n            fs.writeFileSync(outputPath, result.markdown, 'utf8');\r\n\r\n            totalOriginalSize += result.stats.originalHtmlSize;\r\n            totalMarkdownSize += result.stats.markdownSize;\r\n            totalFilesProcessed++;\r\n\r\n            const origKb = (result.stats.originalHtmlSize / 1024).toFixed(1);\r\n            const mdKb = (result.stats.markdownSize / 1024).toFixed(1);\r\n\r\n            // /index.html -> /\r\n            let routeName = file.replace(/\\.html$/, '');\r\n            if (routeName === 'index') routeName = '/';\r\n            else routeName = `/${routeName}`;\r\n\r\n            console.log(\r\n                pc.green(`✓ Optimized`) +\r\n                pc.dim(` ${routeName} `) +\r\n                pc.blue(`[${origKb}KB -> ${mdKb}KB]`)\r\n            );\r\n        } catch (e: any) {\r\n            console.error(pc.red(`✗ Failed to process ${file}: ${e.message}`));\r\n        }\r\n    }\r\n\r\n    console.log(pc.cyan(`\\n[Onto] Finished generation.`));\r\n    console.log(\r\n        pc.bold(\r\n            pc.magenta(`Processed ${totalFilesProcessed} pages. Total Size: ${(totalOriginalSize / 1024).toFixed(1)}KB -> ${(totalMarkdownSize / 1024).toFixed(1)}KB`)\r\n        )\r\n    );\r\n    console.log(pc.dim(`Edge payloads are ready at /public/.onto/*\\n`));\r\n}\r\n\r\nmain().catch(e => {\r\n    console.error(pc.red(`[Onto] Fatal Error: ${e.message}`));\r\n    process.exit(1);\r\n});\r\n","import * as cheerio from 'cheerio';\r\nimport TurndownService from 'turndown';\r\n\r\nconst turndownService = new TurndownService({\r\n    headingStyle: 'atx',\r\n    codeBlockStyle: 'fenced',\r\n});\r\n\r\n// Configure turndown to keep some layout or handle semantic tags differently if needed\r\n\r\nexport interface ExtractionResult {\r\n    markdown: string;\r\n    metadata: {\r\n        title: string;\r\n        description: string;\r\n        jsonLd: any[];\r\n    };\r\n    stats: {\r\n        originalHtmlSize: number;\r\n        markdownSize: number;\r\n        tokenReductionRatio: number;\r\n    };\r\n}\r\n\r\n/**\r\n * Extracts pure semantic markdown and metadata from rendered Next.js HTML strings.\r\n * @param html The raw HTML string.\r\n * @param sourceUrl (Optional) the URL this was generated from, to attach as metadata.\r\n * @returns {ExtractionResult} The extracted payload.\r\n */\r\nexport function extractContent(html: string, sourceUrl: string = 'Generated Output'): ExtractionResult {\r\n    const originalSize = html.length;\r\n\r\n    const $ = cheerio.load(html);\r\n\r\n    // 1. Extract Metadata BEFORE removing structure\r\n    const title = $('title').text() || $('h1').first().text() || 'Untitled Page';\r\n    const description = $('meta[name=\"description\"]').attr('content') || 'No description found.';\r\n\r\n    const jsonLdScripts: any[] = [];\r\n    $('script[type=\"application/ld+json\"]').each((_, el) => {\r\n        try {\r\n            const raw = $(el).html() || '';\r\n            const parsed = JSON.parse(raw);\r\n            jsonLdScripts.push(parsed);\r\n        } catch {\r\n            // ignore bad json\r\n        }\r\n    });\r\n\r\n    // 2. Strip noise (React boilerplate, styles, unnecessary tags)\r\n    $('script, style, noscript, iframe, svg, nav, footer, meta, link, header').remove();\r\n\r\n    // Optionally remove typical Next.js hidden wrappers if they don't contain real content.\r\n    // Next.js uses <div id=\"__next\"> but we mostly just want semantic content.\r\n\r\n    // 3. Find the entry point for content\r\n    // Prefer <main> or <article> over <body>\r\n    let contentHtml = '';\r\n    if ($('main').length > 0) {\r\n        contentHtml = $('main').html() || '';\r\n    } else if ($('article').length > 0) {\r\n        contentHtml = $('article').html() || '';\r\n    } else {\r\n        contentHtml = $('body').html() || '';\r\n    }\r\n\r\n    // 4. Convert to Markdown\r\n    let markdown = turndownService.turndown(contentHtml);\r\n\r\n    // 5. Optionally inject Metadata header\r\n    const headerLines = [\r\n        `# ${title}`,\r\n        `> ${description}`,\r\n        ``,\r\n        `**Source:** ${sourceUrl}`,\r\n        `**Extracted:** ${new Date().toISOString()}`,\r\n        ``,\r\n        `---`,\r\n        ``\r\n    ];\r\n\r\n    let finalMarkdown = headerLines.join('\\n') + markdown;\r\n\r\n    // Add JSON-LD section if exists\r\n    if (jsonLdScripts.length > 0) {\r\n        finalMarkdown += '\\n\\n---\\n## Structured Data (JSON-LD)\\n```json\\n';\r\n        jsonLdScripts.forEach(j => {\r\n            finalMarkdown += JSON.stringify(j, null, 2) + '\\n';\r\n        });\r\n        finalMarkdown += '```\\n';\r\n    }\r\n\r\n    const markdownSize = finalMarkdown.length;\r\n    const tokenReductionRatio = originalSize > 0 ? ((originalSize - markdownSize) / originalSize) * 100 : 0;\r\n\r\n    return {\r\n        markdown: finalMarkdown,\r\n        metadata: {\r\n            title,\r\n            description,\r\n            jsonLd: jsonLdScripts\r\n        },\r\n        stats: {\r\n            originalHtmlSize: originalSize,\r\n            markdownSize,\r\n            tokenReductionRatio\r\n        }\r\n    };\r\n}\r\n\r\nexport async function generateStaticPayloads(nextAppDirDir: string, ontoPublicDir: string) {\r\n    const fs = await import('fs');\r\n    const path = await import('path');\r\n    const { glob } = await import('glob');\r\n\r\n    if (!fs.existsSync(nextAppDirDir)) {\r\n        return;\r\n    }\r\n\r\n    const files = await glob('**/*.html', { cwd: nextAppDirDir });\r\n    if (files.length === 0) return;\r\n\r\n    if (!fs.existsSync(ontoPublicDir)) {\r\n        fs.mkdirSync(ontoPublicDir, { recursive: true });\r\n    }\r\n\r\n    let totalFilesProcessed = 0;\r\n\r\n    for (const file of files) {\r\n        const inputPath = path.join(nextAppDirDir, file);\r\n        const outputPathRelative = file.replace(/\\.html$/, '.md');\r\n        const outputPath = path.join(ontoPublicDir, outputPathRelative);\r\n\r\n        try {\r\n            const htmlContent = fs.readFileSync(inputPath, 'utf8');\r\n\r\n            let routeName = file.replace(/\\.html$/, '');\r\n            if (routeName === 'index') routeName = '/';\r\n            else routeName = `/${routeName}`;\r\n\r\n            const result = extractContent(htmlContent, routeName);\r\n\r\n            const outputDir = path.dirname(outputPath);\r\n            if (!fs.existsSync(outputDir)) {\r\n                fs.mkdirSync(outputDir, { recursive: true });\r\n            }\r\n\r\n            fs.writeFileSync(outputPath, result.markdown, 'utf8');\r\n            totalFilesProcessed++;\r\n        } catch (e: any) {\r\n            console.error(`[Onto] Failed to process ${file}: ${e.message}`);\r\n        }\r\n    }\r\n    console.log(`[Onto] Successfully generated ${totalFilesProcessed} semantic markdown endpoints.`);\r\n}\r\n"],"mappings":";AACA,OAAS,QAAAA,MAAY,OACrB,OAAOC,MAAQ,KACf,OAAOC,MAAU,OACjB,OAAOC,MAAQ,aCJf,UAAYC,MAAa,UACzB,OAAOC,MAAqB,WAE5B,IAAMC,EAAkB,IAAID,EAAgB,CACxC,aAAc,MACd,eAAgB,QACpB,CAAC,EAwBM,SAASE,EAAeC,EAAcC,EAAoB,mBAAsC,CACnG,IAAMC,EAAeF,EAAK,OAEpBG,EAAY,OAAKH,CAAI,EAGrBI,EAAQD,EAAE,OAAO,EAAE,KAAK,GAAKA,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,GAAK,gBACvDE,EAAcF,EAAE,0BAA0B,EAAE,KAAK,SAAS,GAAK,wBAE/DG,EAAuB,CAAC,EAC9BH,EAAE,oCAAoC,EAAE,KAAK,CAACI,EAAGC,IAAO,CACpD,GAAI,CACA,IAAMC,EAAMN,EAAEK,CAAE,EAAE,KAAK,GAAK,GACtBE,EAAS,KAAK,MAAMD,CAAG,EAC7BH,EAAc,KAAKI,CAAM,CAC7B,MAAQ,CAER,CACJ,CAAC,EAGDP,EAAE,uEAAuE,EAAE,OAAO,EAOlF,IAAIQ,EAAc,GACdR,EAAE,MAAM,EAAE,OAAS,EACnBQ,EAAcR,EAAE,MAAM,EAAE,KAAK,GAAK,GAC3BA,EAAE,SAAS,EAAE,OAAS,EAC7BQ,EAAcR,EAAE,SAAS,EAAE,KAAK,GAAK,GAErCQ,EAAcR,EAAE,MAAM,EAAE,KAAK,GAAK,GAItC,IAAIS,EAAWd,EAAgB,SAASa,CAAW,EAc/CE,EAXgB,CAChB,KAAKT,CAAK,GACV,KAAKC,CAAW,GAChB,GACA,eAAeJ,CAAS,GACxB,kBAAkB,IAAI,KAAK,EAAE,YAAY,CAAC,GAC1C,GACA,MACA,EACJ,EAEgC,KAAK;AAAA,CAAI,EAAIW,EAGzCN,EAAc,OAAS,IACvBO,GAAiB;AAAA;AAAA;AAAA;AAAA;AAAA,EACjBP,EAAc,QAAQQ,GAAK,CACvBD,GAAiB,KAAK,UAAUC,EAAG,KAAM,CAAC,EAAI;AAAA,CAClD,CAAC,EACDD,GAAiB,SAGrB,IAAME,EAAeF,EAAc,OAC7BG,EAAsBd,EAAe,GAAMA,EAAea,GAAgBb,EAAgB,IAAM,EAEtG,MAAO,CACH,SAAUW,EACV,SAAU,CACN,MAAAT,EACA,YAAAC,EACA,OAAQC,CACZ,EACA,MAAO,CACH,iBAAkBJ,EAClB,aAAAa,EACA,oBAAAC,CACJ,CACJ,CACJ,CDtGA,eAAeC,GAAO,CAClB,QAAQ,IAAIC,EAAG,KAAK;AAAA,8CAAiD,CAAC,EAEtE,IAAMC,EAAM,QAAQ,IAAI,EAClBC,EAAgBC,EAAK,KAAKF,EAAK,kBAAkB,EACjDG,EAAgBD,EAAK,KAAKF,EAAK,cAAc,EAEnD,GAAI,CAACI,EAAG,WAAWH,CAAa,EAAG,CAC/B,QAAQ,IAAIF,EAAG,OAAO,+CAA+CE,CAAa,EAAE,CAAC,EACrF,QAAQ,IAAIF,EAAG,OAAO,gFAAgF,CAAC,EACvG,MACJ,CAGA,IAAMM,EAAQ,MAAMC,EAAK,YAAa,CAAE,IAAKL,CAAc,CAAC,EAE5D,GAAII,EAAM,SAAW,EAAG,CACpB,QAAQ,IAAIN,EAAG,OAAO,+CAA+C,CAAC,EACtE,MACJ,CAGKK,EAAG,WAAWD,CAAa,GAC5BC,EAAG,UAAUD,EAAe,CAAE,UAAW,EAAK,CAAC,EAGnD,IAAII,EAAoB,EACpBC,EAAoB,EACpBC,EAAsB,EAE1B,QAAWC,KAAQL,EAAO,CACtB,IAAMM,EAAYT,EAAK,KAAKD,EAAeS,CAAI,EAG3CE,EAAqBF,EAAK,QAAQ,UAAW,KAAK,EAEhDG,EAAaX,EAAK,KAAKC,EAAeS,CAAkB,EAE9D,GAAI,CACA,IAAME,EAAcV,EAAG,aAAaO,EAAW,MAAM,EAE/CI,EAASC,EAAeF,EAAa,IAAIF,EAAmB,QAAQ,QAAS,EAAE,CAAC,EAAE,EAGlFK,EAAYf,EAAK,QAAQW,CAAU,EACpCT,EAAG,WAAWa,CAAS,GACxBb,EAAG,UAAUa,EAAW,CAAE,UAAW,EAAK,CAAC,EAG/Cb,EAAG,cAAcS,EAAYE,EAAO,SAAU,MAAM,EAEpDR,GAAqBQ,EAAO,MAAM,iBAClCP,GAAqBO,EAAO,MAAM,aAClCN,IAEA,IAAMS,GAAUH,EAAO,MAAM,iBAAmB,MAAM,QAAQ,CAAC,EACzDI,GAAQJ,EAAO,MAAM,aAAe,MAAM,QAAQ,CAAC,EAGrDK,EAAYV,EAAK,QAAQ,UAAW,EAAE,EACtCU,IAAc,QAASA,EAAY,IAClCA,EAAY,IAAIA,CAAS,GAE9B,QAAQ,IACJrB,EAAG,MAAM,kBAAa,EACtBA,EAAG,IAAI,IAAIqB,CAAS,GAAG,EACvBrB,EAAG,KAAK,IAAImB,CAAM,SAASC,CAAI,KAAK,CACxC,CACJ,OAASE,EAAQ,CACb,QAAQ,MAAMtB,EAAG,IAAI,4BAAuBW,CAAI,KAAKW,EAAE,OAAO,EAAE,CAAC,CACrE,CACJ,CAEA,QAAQ,IAAItB,EAAG,KAAK;AAAA,4BAA+B,CAAC,EACpD,QAAQ,IACJA,EAAG,KACCA,EAAG,QAAQ,aAAaU,CAAmB,wBAAwBF,EAAoB,MAAM,QAAQ,CAAC,CAAC,UAAUC,EAAoB,MAAM,QAAQ,CAAC,CAAC,IAAI,CAC7J,CACJ,EACA,QAAQ,IAAIT,EAAG,IAAI;AAAA,CAA8C,CAAC,CACtE,CAEAD,EAAK,EAAE,MAAMuB,GAAK,CACd,QAAQ,MAAMtB,EAAG,IAAI,uBAAuBsB,EAAE,OAAO,EAAE,CAAC,EACxD,QAAQ,KAAK,CAAC,CAClB,CAAC","names":["glob","fs","path","pc","cheerio","TurndownService","turndownService","extractContent","html","sourceUrl","originalSize","$","title","description","jsonLdScripts","_","el","raw","parsed","contentHtml","markdown","finalMarkdown","j","markdownSize","tokenReductionRatio","main","pc","cwd","nextAppDirDir","path","ontoPublicDir","fs","files","glob","totalOriginalSize","totalMarkdownSize","totalFilesProcessed","file","inputPath","outputPathRelative","outputPath","htmlContent","result","extractContent","outputDir","origKb","mdKb","routeName","e"]}

package/dist/index.d.mts

@@ -0,0 +1,22 @@
+interface ExtractionResult {
+    markdown: string;
+    metadata: {
+        title: string;
+        description: string;
+        jsonLd: any[];
+    };
+    stats: {
+        originalHtmlSize: number;
+        markdownSize: number;
+        tokenReductionRatio: number;
+    };
+}
+/**
+ * Extracts pure semantic markdown and metadata from rendered Next.js HTML strings.
+ * @param html The raw HTML string.
+ * @param sourceUrl (Optional) the URL this was generated from, to attach as metadata.
+ * @returns {ExtractionResult} The extracted payload.
+ */
+declare function extractContent(html: string, sourceUrl?: string): ExtractionResult;
+
+export { extractContent };

package/dist/index.d.ts

@@ -0,0 +1,22 @@
+interface ExtractionResult {
+    markdown: string;
+    metadata: {
+        title: string;
+        description: string;
+        jsonLd: any[];
+    };
+    stats: {
+        originalHtmlSize: number;
+        markdownSize: number;
+        tokenReductionRatio: number;
+    };
+}
+/**
+ * Extracts pure semantic markdown and metadata from rendered Next.js HTML strings.
+ * @param html The raw HTML string.
+ * @param sourceUrl (Optional) the URL this was generated from, to attach as metadata.
+ * @returns {ExtractionResult} The extracted payload.
+ */
+declare function extractContent(html: string, sourceUrl?: string): ExtractionResult;
+
+export { extractContent };

package/dist/index.js

@@ -0,0 +1,9 @@
+"use strict";var $=Object.create;var c=Object.defineProperty;var j=Object.getOwnPropertyDescriptor;var v=Object.getOwnPropertyNames;var O=Object.getPrototypeOf,R=Object.prototype.hasOwnProperty;var L=(t,n)=>{for(var o in n)c(t,o,{get:n[o],enumerable:!0})},u=(t,n,o,e)=>{if(n&&typeof n=="object"||typeof n=="function")for(let r of v(n))!R.call(t,r)&&r!==o&&c(t,r,{get:()=>n[r],enumerable:!(e=j(n,r))||e.enumerable});return t};var f=(t,n,o)=>(o=t!=null?$(O(t)):{},u(n||!t||!t.__esModule?c(o,"default",{value:t,enumerable:!0}):o,t)),z=t=>u(c({},"__esModule",{value:!0}),t);var E={};L(E,{extractContent:()=>g});module.exports=z(E);var h=f(require("cheerio")),p=f(require("turndown")),N=new p.default({headingStyle:"atx",codeBlockStyle:"fenced"});function g(t,n="Generated Output"){let o=t.length,e=h.load(t),r=e("title").text()||e("h1").first().text()||"Untitled Page",l=e('meta[name="description"]').attr("content")||"No description found.",a=[];e('script[type="application/ld+json"]').each((m,w)=>{try{let k=e(w).html()||"",x=JSON.parse(k);a.push(x)}catch{}}),e("script, style, noscript, iframe, svg, nav, footer, meta, link, header").remove();let s="";e("main").length>0?s=e("main").html()||"":e("article").length>0?s=e("article").html()||"":s=e("body").html()||"";let S=N.turndown(s),i=[`# ${r}`,`> ${l}`,"",`**Source:** ${n}`,`**Extracted:** ${new Date().toISOString()}`,"","---",""].join(`
+`)+S;a.length>0&&(i+=`
+
+---
+## Structured Data (JSON-LD)
+\`\`\`json
+`,a.forEach(m=>{i+=JSON.stringify(m,null,2)+`
+`}),i+="```\n");let d=i.length,y=o>0?(o-d)/o*100:0;return{markdown:i,metadata:{title:r,description:l,jsonLd:a},stats:{originalHtmlSize:o,markdownSize:d,tokenReductionRatio:y}}}0&&(module.exports={extractContent});
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/index.ts","../src/extractor.ts"],"sourcesContent":["// We cannot use Webpack plugins reliably in Next.js Turbopack due to WorkerError restrictions.\r\n// Users must instead run `npx onto-next` as a postbuild script.\r\nexport { extractContent } from './extractor';\r\n","import * as cheerio from 'cheerio';\r\nimport TurndownService from 'turndown';\r\n\r\nconst turndownService = new TurndownService({\r\n    headingStyle: 'atx',\r\n    codeBlockStyle: 'fenced',\r\n});\r\n\r\n// Configure turndown to keep some layout or handle semantic tags differently if needed\r\n\r\nexport interface ExtractionResult {\r\n    markdown: string;\r\n    metadata: {\r\n        title: string;\r\n        description: string;\r\n        jsonLd: any[];\r\n    };\r\n    stats: {\r\n        originalHtmlSize: number;\r\n        markdownSize: number;\r\n        tokenReductionRatio: number;\r\n    };\r\n}\r\n\r\n/**\r\n * Extracts pure semantic markdown and metadata from rendered Next.js HTML strings.\r\n * @param html The raw HTML string.\r\n * @param sourceUrl (Optional) the URL this was generated from, to attach as metadata.\r\n * @returns {ExtractionResult} The extracted payload.\r\n */\r\nexport function extractContent(html: string, sourceUrl: string = 'Generated Output'): ExtractionResult {\r\n    const originalSize = html.length;\r\n\r\n    const $ = cheerio.load(html);\r\n\r\n    // 1. Extract Metadata BEFORE removing structure\r\n    const title = $('title').text() || $('h1').first().text() || 'Untitled Page';\r\n    const description = $('meta[name=\"description\"]').attr('content') || 'No description found.';\r\n\r\n    const jsonLdScripts: any[] = [];\r\n    $('script[type=\"application/ld+json\"]').each((_, el) => {\r\n        try {\r\n            const raw = $(el).html() || '';\r\n            const parsed = JSON.parse(raw);\r\n            jsonLdScripts.push(parsed);\r\n        } catch {\r\n            // ignore bad json\r\n        }\r\n    });\r\n\r\n    // 2. Strip noise (React boilerplate, styles, unnecessary tags)\r\n    $('script, style, noscript, iframe, svg, nav, footer, meta, link, header').remove();\r\n\r\n    // Optionally remove typical Next.js hidden wrappers if they don't contain real content.\r\n    // Next.js uses <div id=\"__next\"> but we mostly just want semantic content.\r\n\r\n    // 3. Find the entry point for content\r\n    // Prefer <main> or <article> over <body>\r\n    let contentHtml = '';\r\n    if ($('main').length > 0) {\r\n        contentHtml = $('main').html() || '';\r\n    } else if ($('article').length > 0) {\r\n        contentHtml = $('article').html() || '';\r\n    } else {\r\n        contentHtml = $('body').html() || '';\r\n    }\r\n\r\n    // 4. Convert to Markdown\r\n    let markdown = turndownService.turndown(contentHtml);\r\n\r\n    // 5. Optionally inject Metadata header\r\n    const headerLines = [\r\n        `# ${title}`,\r\n        `> ${description}`,\r\n        ``,\r\n        `**Source:** ${sourceUrl}`,\r\n        `**Extracted:** ${new Date().toISOString()}`,\r\n        ``,\r\n        `---`,\r\n        ``\r\n    ];\r\n\r\n    let finalMarkdown = headerLines.join('\\n') + markdown;\r\n\r\n    // Add JSON-LD section if exists\r\n    if (jsonLdScripts.length > 0) {\r\n        finalMarkdown += '\\n\\n---\\n## Structured Data (JSON-LD)\\n```json\\n';\r\n        jsonLdScripts.forEach(j => {\r\n            finalMarkdown += JSON.stringify(j, null, 2) + '\\n';\r\n        });\r\n        finalMarkdown += '```\\n';\r\n    }\r\n\r\n    const markdownSize = finalMarkdown.length;\r\n    const tokenReductionRatio = originalSize > 0 ? ((originalSize - markdownSize) / originalSize) * 100 : 0;\r\n\r\n    return {\r\n        markdown: finalMarkdown,\r\n        metadata: {\r\n            title,\r\n            description,\r\n            jsonLd: jsonLdScripts\r\n        },\r\n        stats: {\r\n            originalHtmlSize: originalSize,\r\n            markdownSize,\r\n            tokenReductionRatio\r\n        }\r\n    };\r\n}\r\n\r\nexport async function generateStaticPayloads(nextAppDirDir: string, ontoPublicDir: string) {\r\n    const fs = await import('fs');\r\n    const path = await import('path');\r\n    const { glob } = await import('glob');\r\n\r\n    if (!fs.existsSync(nextAppDirDir)) {\r\n        return;\r\n    }\r\n\r\n    const files = await glob('**/*.html', { cwd: nextAppDirDir });\r\n    if (files.length === 0) return;\r\n\r\n    if (!fs.existsSync(ontoPublicDir)) {\r\n        fs.mkdirSync(ontoPublicDir, { recursive: true });\r\n    }\r\n\r\n    let totalFilesProcessed = 0;\r\n\r\n    for (const file of files) {\r\n        const inputPath = path.join(nextAppDirDir, file);\r\n        const outputPathRelative = file.replace(/\\.html$/, '.md');\r\n        const outputPath = path.join(ontoPublicDir, outputPathRelative);\r\n\r\n        try {\r\n            const htmlContent = fs.readFileSync(inputPath, 'utf8');\r\n\r\n            let routeName = file.replace(/\\.html$/, '');\r\n            if (routeName === 'index') routeName = '/';\r\n            else routeName = `/${routeName}`;\r\n\r\n            const result = extractContent(htmlContent, routeName);\r\n\r\n            const outputDir = path.dirname(outputPath);\r\n            if (!fs.existsSync(outputDir)) {\r\n                fs.mkdirSync(outputDir, { recursive: true });\r\n            }\r\n\r\n            fs.writeFileSync(outputPath, result.markdown, 'utf8');\r\n            totalFilesProcessed++;\r\n        } catch (e: any) {\r\n            console.error(`[Onto] Failed to process ${file}: ${e.message}`);\r\n        }\r\n    }\r\n    console.log(`[Onto] Successfully generated ${totalFilesProcessed} semantic markdown endpoints.`);\r\n}\r\n"],"mappings":"0jBAAA,IAAAA,EAAA,GAAAC,EAAAD,EAAA,oBAAAE,IAAA,eAAAC,EAAAH,GCAA,IAAAI,EAAyB,sBACzBC,EAA4B,uBAEtBC,EAAkB,IAAI,EAAAC,QAAgB,CACxC,aAAc,MACd,eAAgB,QACpB,CAAC,EAwBM,SAASC,EAAeC,EAAcC,EAAoB,mBAAsC,CACnG,IAAMC,EAAeF,EAAK,OAEpBG,EAAY,OAAKH,CAAI,EAGrBI,EAAQD,EAAE,OAAO,EAAE,KAAK,GAAKA,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,GAAK,gBACvDE,EAAcF,EAAE,0BAA0B,EAAE,KAAK,SAAS,GAAK,wBAE/DG,EAAuB,CAAC,EAC9BH,EAAE,oCAAoC,EAAE,KAAK,CAACI,EAAGC,IAAO,CACpD,GAAI,CACA,IAAMC,EAAMN,EAAEK,CAAE,EAAE,KAAK,GAAK,GACtBE,EAAS,KAAK,MAAMD,CAAG,EAC7BH,EAAc,KAAKI,CAAM,CAC7B,MAAQ,CAER,CACJ,CAAC,EAGDP,EAAE,uEAAuE,EAAE,OAAO,EAOlF,IAAIQ,EAAc,GACdR,EAAE,MAAM,EAAE,OAAS,EACnBQ,EAAcR,EAAE,MAAM,EAAE,KAAK,GAAK,GAC3BA,EAAE,SAAS,EAAE,OAAS,EAC7BQ,EAAcR,EAAE,SAAS,EAAE,KAAK,GAAK,GAErCQ,EAAcR,EAAE,MAAM,EAAE,KAAK,GAAK,GAItC,IAAIS,EAAWf,EAAgB,SAASc,CAAW,EAc/CE,EAXgB,CAChB,KAAKT,CAAK,GACV,KAAKC,CAAW,GAChB,GACA,eAAeJ,CAAS,GACxB,kBAAkB,IAAI,KAAK,EAAE,YAAY,CAAC,GAC1C,GACA,MACA,EACJ,EAEgC,KAAK;AAAA,CAAI,EAAIW,EAGzCN,EAAc,OAAS,IACvBO,GAAiB;AAAA;AAAA;AAAA;AAAA;AAAA,EACjBP,EAAc,QAAQQ,GAAK,CACvBD,GAAiB,KAAK,UAAUC,EAAG,KAAM,CAAC,EAAI;AAAA,CAClD,CAAC,EACDD,GAAiB,SAGrB,IAAME,EAAeF,EAAc,OAC7BG,EAAsBd,EAAe,GAAMA,EAAea,GAAgBb,EAAgB,IAAM,EAEtG,MAAO,CACH,SAAUW,EACV,SAAU,CACN,MAAAT,EACA,YAAAC,EACA,OAAQC,CACZ,EACA,MAAO,CACH,iBAAkBJ,EAClB,aAAAa,EACA,oBAAAC,CACJ,CACJ,CACJ","names":["index_exports","__export","extractContent","__toCommonJS","cheerio","import_turndown","turndownService","TurndownService","extractContent","html","sourceUrl","originalSize","$","title","description","jsonLdScripts","_","el","raw","parsed","contentHtml","markdown","finalMarkdown","j","markdownSize","tokenReductionRatio"]}

package/dist/index.mjs

@@ -0,0 +1,9 @@
+import*as d from"cheerio";import S from"turndown";var y=new S({headingStyle:"atx",codeBlockStyle:"fenced"});function w(i,m="Generated Output"){let n=i.length,t=d.load(i),a=t("title").text()||t("h1").first().text()||"Untitled Page",s=t('meta[name="description"]').attr("content")||"No description found.",o=[];t('script[type="application/ld+json"]').each((l,h)=>{try{let p=t(h).html()||"",g=JSON.parse(p);o.push(g)}catch{}}),t("script, style, noscript, iframe, svg, nav, footer, meta, link, header").remove();let r="";t("main").length>0?r=t("main").html()||"":t("article").length>0?r=t("article").html()||"":r=t("body").html()||"";let u=y.turndown(r),e=[`# ${a}`,`> ${s}`,"",`**Source:** ${m}`,`**Extracted:** ${new Date().toISOString()}`,"","---",""].join(`
+`)+u;o.length>0&&(e+=`
+
+---
+## Structured Data (JSON-LD)
+\`\`\`json
+`,o.forEach(l=>{e+=JSON.stringify(l,null,2)+`
+`}),e+="```\n");let c=e.length,f=n>0?(n-c)/n*100:0;return{markdown:e,metadata:{title:a,description:s,jsonLd:o},stats:{originalHtmlSize:n,markdownSize:c,tokenReductionRatio:f}}}export{w as extractContent};
+//# sourceMappingURL=index.mjs.map

package/dist/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/extractor.ts"],"sourcesContent":["import * as cheerio from 'cheerio';\r\nimport TurndownService from 'turndown';\r\n\r\nconst turndownService = new TurndownService({\r\n    headingStyle: 'atx',\r\n    codeBlockStyle: 'fenced',\r\n});\r\n\r\n// Configure turndown to keep some layout or handle semantic tags differently if needed\r\n\r\nexport interface ExtractionResult {\r\n    markdown: string;\r\n    metadata: {\r\n        title: string;\r\n        description: string;\r\n        jsonLd: any[];\r\n    };\r\n    stats: {\r\n        originalHtmlSize: number;\r\n        markdownSize: number;\r\n        tokenReductionRatio: number;\r\n    };\r\n}\r\n\r\n/**\r\n * Extracts pure semantic markdown and metadata from rendered Next.js HTML strings.\r\n * @param html The raw HTML string.\r\n * @param sourceUrl (Optional) the URL this was generated from, to attach as metadata.\r\n * @returns {ExtractionResult} The extracted payload.\r\n */\r\nexport function extractContent(html: string, sourceUrl: string = 'Generated Output'): ExtractionResult {\r\n    const originalSize = html.length;\r\n\r\n    const $ = cheerio.load(html);\r\n\r\n    // 1. Extract Metadata BEFORE removing structure\r\n    const title = $('title').text() || $('h1').first().text() || 'Untitled Page';\r\n    const description = $('meta[name=\"description\"]').attr('content') || 'No description found.';\r\n\r\n    const jsonLdScripts: any[] = [];\r\n    $('script[type=\"application/ld+json\"]').each((_, el) => {\r\n        try {\r\n            const raw = $(el).html() || '';\r\n            const parsed = JSON.parse(raw);\r\n            jsonLdScripts.push(parsed);\r\n        } catch {\r\n            // ignore bad json\r\n        }\r\n    });\r\n\r\n    // 2. Strip noise (React boilerplate, styles, unnecessary tags)\r\n    $('script, style, noscript, iframe, svg, nav, footer, meta, link, header').remove();\r\n\r\n    // Optionally remove typical Next.js hidden wrappers if they don't contain real content.\r\n    // Next.js uses <div id=\"__next\"> but we mostly just want semantic content.\r\n\r\n    // 3. Find the entry point for content\r\n    // Prefer <main> or <article> over <body>\r\n    let contentHtml = '';\r\n    if ($('main').length > 0) {\r\n        contentHtml = $('main').html() || '';\r\n    } else if ($('article').length > 0) {\r\n        contentHtml = $('article').html() || '';\r\n    } else {\r\n        contentHtml = $('body').html() || '';\r\n    }\r\n\r\n    // 4. Convert to Markdown\r\n    let markdown = turndownService.turndown(contentHtml);\r\n\r\n    // 5. Optionally inject Metadata header\r\n    const headerLines = [\r\n        `# ${title}`,\r\n        `> ${description}`,\r\n        ``,\r\n        `**Source:** ${sourceUrl}`,\r\n        `**Extracted:** ${new Date().toISOString()}`,\r\n        ``,\r\n        `---`,\r\n        ``\r\n    ];\r\n\r\n    let finalMarkdown = headerLines.join('\\n') + markdown;\r\n\r\n    // Add JSON-LD section if exists\r\n    if (jsonLdScripts.length > 0) {\r\n        finalMarkdown += '\\n\\n---\\n## Structured Data (JSON-LD)\\n```json\\n';\r\n        jsonLdScripts.forEach(j => {\r\n            finalMarkdown += JSON.stringify(j, null, 2) + '\\n';\r\n        });\r\n        finalMarkdown += '```\\n';\r\n    }\r\n\r\n    const markdownSize = finalMarkdown.length;\r\n    const tokenReductionRatio = originalSize > 0 ? ((originalSize - markdownSize) / originalSize) * 100 : 0;\r\n\r\n    return {\r\n        markdown: finalMarkdown,\r\n        metadata: {\r\n            title,\r\n            description,\r\n            jsonLd: jsonLdScripts\r\n        },\r\n        stats: {\r\n            originalHtmlSize: originalSize,\r\n            markdownSize,\r\n            tokenReductionRatio\r\n        }\r\n    };\r\n}\r\n\r\nexport async function generateStaticPayloads(nextAppDirDir: string, ontoPublicDir: string) {\r\n    const fs = await import('fs');\r\n    const path = await import('path');\r\n    const { glob } = await import('glob');\r\n\r\n    if (!fs.existsSync(nextAppDirDir)) {\r\n        return;\r\n    }\r\n\r\n    const files = await glob('**/*.html', { cwd: nextAppDirDir });\r\n    if (files.length === 0) return;\r\n\r\n    if (!fs.existsSync(ontoPublicDir)) {\r\n        fs.mkdirSync(ontoPublicDir, { recursive: true });\r\n    }\r\n\r\n    let totalFilesProcessed = 0;\r\n\r\n    for (const file of files) {\r\n        const inputPath = path.join(nextAppDirDir, file);\r\n        const outputPathRelative = file.replace(/\\.html$/, '.md');\r\n        const outputPath = path.join(ontoPublicDir, outputPathRelative);\r\n\r\n        try {\r\n            const htmlContent = fs.readFileSync(inputPath, 'utf8');\r\n\r\n            let routeName = file.replace(/\\.html$/, '');\r\n            if (routeName === 'index') routeName = '/';\r\n            else routeName = `/${routeName}`;\r\n\r\n            const result = extractContent(htmlContent, routeName);\r\n\r\n            const outputDir = path.dirname(outputPath);\r\n            if (!fs.existsSync(outputDir)) {\r\n                fs.mkdirSync(outputDir, { recursive: true });\r\n            }\r\n\r\n            fs.writeFileSync(outputPath, result.markdown, 'utf8');\r\n            totalFilesProcessed++;\r\n        } catch (e: any) {\r\n            console.error(`[Onto] Failed to process ${file}: ${e.message}`);\r\n        }\r\n    }\r\n    console.log(`[Onto] Successfully generated ${totalFilesProcessed} semantic markdown endpoints.`);\r\n}\r\n"],"mappings":"AAAA,UAAYA,MAAa,UACzB,OAAOC,MAAqB,WAE5B,IAAMC,EAAkB,IAAID,EAAgB,CACxC,aAAc,MACd,eAAgB,QACpB,CAAC,EAwBM,SAASE,EAAeC,EAAcC,EAAoB,mBAAsC,CACnG,IAAMC,EAAeF,EAAK,OAEpBG,EAAY,OAAKH,CAAI,EAGrBI,EAAQD,EAAE,OAAO,EAAE,KAAK,GAAKA,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,GAAK,gBACvDE,EAAcF,EAAE,0BAA0B,EAAE,KAAK,SAAS,GAAK,wBAE/DG,EAAuB,CAAC,EAC9BH,EAAE,oCAAoC,EAAE,KAAK,CAACI,EAAGC,IAAO,CACpD,GAAI,CACA,IAAMC,EAAMN,EAAEK,CAAE,EAAE,KAAK,GAAK,GACtBE,EAAS,KAAK,MAAMD,CAAG,EAC7BH,EAAc,KAAKI,CAAM,CAC7B,MAAQ,CAER,CACJ,CAAC,EAGDP,EAAE,uEAAuE,EAAE,OAAO,EAOlF,IAAIQ,EAAc,GACdR,EAAE,MAAM,EAAE,OAAS,EACnBQ,EAAcR,EAAE,MAAM,EAAE,KAAK,GAAK,GAC3BA,EAAE,SAAS,EAAE,OAAS,EAC7BQ,EAAcR,EAAE,SAAS,EAAE,KAAK,GAAK,GAErCQ,EAAcR,EAAE,MAAM,EAAE,KAAK,GAAK,GAItC,IAAIS,EAAWd,EAAgB,SAASa,CAAW,EAc/CE,EAXgB,CAChB,KAAKT,CAAK,GACV,KAAKC,CAAW,GAChB,GACA,eAAeJ,CAAS,GACxB,kBAAkB,IAAI,KAAK,EAAE,YAAY,CAAC,GAC1C,GACA,MACA,EACJ,EAEgC,KAAK;AAAA,CAAI,EAAIW,EAGzCN,EAAc,OAAS,IACvBO,GAAiB;AAAA;AAAA;AAAA;AAAA;AAAA,EACjBP,EAAc,QAAQQ,GAAK,CACvBD,GAAiB,KAAK,UAAUC,EAAG,KAAM,CAAC,EAAI;AAAA,CAClD,CAAC,EACDD,GAAiB,SAGrB,IAAME,EAAeF,EAAc,OAC7BG,EAAsBd,EAAe,GAAMA,EAAea,GAAgBb,EAAgB,IAAM,EAEtG,MAAO,CACH,SAAUW,EACV,SAAU,CACN,MAAAT,EACA,YAAAC,EACA,OAAQC,CACZ,EACA,MAAO,CACH,iBAAkBJ,EAClB,aAAAa,EACA,oBAAAC,CACJ,CACJ,CACJ","names":["cheerio","TurndownService","turndownService","extractContent","html","sourceUrl","originalSize","$","title","description","jsonLdScripts","_","el","raw","parsed","contentHtml","markdown","finalMarkdown","j","markdownSize","tokenReductionRatio"]}

package/dist/middleware.d.mts

@@ -0,0 +1,5 @@
+import { NextRequest, NextResponse } from 'next/server';
+
+declare function ontoMiddleware(request: NextRequest): NextResponse<unknown>;
+
+export { ontoMiddleware };

package/dist/middleware.d.ts

@@ -0,0 +1,5 @@
+import { NextRequest, NextResponse } from 'next/server';
+
+declare function ontoMiddleware(request: NextRequest): NextResponse<unknown>;
+
+export { ontoMiddleware };

package/dist/middleware.js

@@ -0,0 +1,2 @@
+"use strict";var c=Object.defineProperty;var d=Object.getOwnPropertyDescriptor;var l=Object.getOwnPropertyNames;var u=Object.prototype.hasOwnProperty;var h=(e,t)=>{for(var s in t)c(e,s,{get:t[s],enumerable:!0})},p=(e,t,s,r)=>{if(t&&typeof t=="object"||typeof t=="function")for(let o of l(t))!u.call(e,o)&&o!==s&&c(e,o,{get:()=>t[o],enumerable:!(r=d(t,o))||r.enumerable});return e};var x=e=>p(c({},"__esModule",{value:!0}),e);var B={};h(B,{ontoMiddleware:()=>f});module.exports=x(B);var i=require("next/server"),m=["GPTBot","ChatGPT-User","ClaudeBot","Claude-Web","anthropic-ai","PerplexityBot","OAI-SearchBot","GoogleExtended"];function f(e){let t=e.headers.get("user-agent")||"",s=e.headers.get("accept")||"",r=m.some(a=>t.includes(a)),o=s.includes("text/markdown");if(r||o){let a=e.nextUrl.clone();if(a.pathname.startsWith("/_next")||a.pathname.includes("."))return i.NextResponse.next();let n=a.pathname;return(n==="/"||n==="")&&(n="/index"),n.endsWith("/")&&n!=="/"&&(n=n.slice(0,-1)),a.pathname=`/.onto${n}.md`,i.NextResponse.rewrite(a)}return i.NextResponse.next()}0&&(module.exports={ontoMiddleware});
+//# sourceMappingURL=middleware.js.map

package/dist/middleware.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/middleware.ts"],"sourcesContent":["import { NextRequest, NextResponse } from 'next/server';\r\n\r\nconst AI_BOT_USER_AGENTS = [\r\n    'GPTBot',\r\n    'ChatGPT-User',\r\n    'ClaudeBot',\r\n    'Claude-Web',\r\n    'anthropic-ai',\r\n    'PerplexityBot',\r\n    'OAI-SearchBot',\r\n    'GoogleExtended',\r\n];\r\n\r\nexport function ontoMiddleware(request: NextRequest) {\r\n    const userAgent = request.headers.get('user-agent') || '';\r\n    const accept = request.headers.get('accept') || '';\r\n\r\n    const isAiBot = AI_BOT_USER_AGENTS.some(bot => userAgent.includes(bot));\r\n    const isMarkdownRequested = accept.includes('text/markdown');\r\n\r\n    // If traffic is identified as an AI Bot, rewrite the URL\r\n    if (isAiBot || isMarkdownRequested) {\r\n        const url = request.nextUrl.clone();\r\n\r\n        // Ignore internal next.js requests & static assets\r\n        if (url.pathname.startsWith('/_next') || url.pathname.includes('.')) {\r\n            return NextResponse.next();\r\n        }\r\n\r\n        // Determine the corresponding payload path\r\n        let payloadPath = url.pathname;\r\n        if (payloadPath === '/' || payloadPath === '') {\r\n            payloadPath = '/index';\r\n        }\r\n\r\n        // Strip trailing slash if present\r\n        if (payloadPath.endsWith('/') && payloadPath !== '/') {\r\n            payloadPath = payloadPath.slice(0, -1);\r\n        }\r\n\r\n        url.pathname = `/.onto${payloadPath}.md`;\r\n\r\n        // Rewrite implicitly serves the target URL transparently to the client.\r\n        return NextResponse.rewrite(url);\r\n    }\r\n\r\n    return NextResponse.next();\r\n}\r\n"],"mappings":"yaAAA,IAAAA,EAAA,GAAAC,EAAAD,EAAA,oBAAAE,IAAA,eAAAC,EAAAH,GAAA,IAAAI,EAA0C,uBAEpCC,EAAqB,CACvB,SACA,eACA,YACA,aACA,eACA,gBACA,gBACA,gBACJ,EAEO,SAASH,EAAeI,EAAsB,CACjD,IAAMC,EAAYD,EAAQ,QAAQ,IAAI,YAAY,GAAK,GACjDE,EAASF,EAAQ,QAAQ,IAAI,QAAQ,GAAK,GAE1CG,EAAUJ,EAAmB,KAAKK,GAAOH,EAAU,SAASG,CAAG,CAAC,EAChEC,EAAsBH,EAAO,SAAS,eAAe,EAG3D,GAAIC,GAAWE,EAAqB,CAChC,IAAMC,EAAMN,EAAQ,QAAQ,MAAM,EAGlC,GAAIM,EAAI,SAAS,WAAW,QAAQ,GAAKA,EAAI,SAAS,SAAS,GAAG,EAC9D,OAAO,eAAa,KAAK,EAI7B,IAAIC,EAAcD,EAAI,SACtB,OAAIC,IAAgB,KAAOA,IAAgB,MACvCA,EAAc,UAIdA,EAAY,SAAS,GAAG,GAAKA,IAAgB,MAC7CA,EAAcA,EAAY,MAAM,EAAG,EAAE,GAGzCD,EAAI,SAAW,SAASC,CAAW,MAG5B,eAAa,QAAQD,CAAG,CACnC,CAEA,OAAO,eAAa,KAAK,CAC7B","names":["middleware_exports","__export","ontoMiddleware","__toCommonJS","import_server","AI_BOT_USER_AGENTS","request","userAgent","accept","isAiBot","bot","isMarkdownRequested","url","payloadPath"]}

package/dist/middleware.mjs

@@ -0,0 +1,2 @@
+import{NextResponse as o}from"next/server";var c=["GPTBot","ChatGPT-User","ClaudeBot","Claude-Web","anthropic-ai","PerplexityBot","OAI-SearchBot","GoogleExtended"];function u(n){let a=n.headers.get("user-agent")||"",s=n.headers.get("accept")||"",r=c.some(t=>a.includes(t)),i=s.includes("text/markdown");if(r||i){let t=n.nextUrl.clone();if(t.pathname.startsWith("/_next")||t.pathname.includes("."))return o.next();let e=t.pathname;return(e==="/"||e==="")&&(e="/index"),e.endsWith("/")&&e!=="/"&&(e=e.slice(0,-1)),t.pathname=`/.onto${e}.md`,o.rewrite(t)}return o.next()}export{u as ontoMiddleware};
+//# sourceMappingURL=middleware.mjs.map

package/dist/middleware.mjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/middleware.ts"],"sourcesContent":["import { NextRequest, NextResponse } from 'next/server';\r\n\r\nconst AI_BOT_USER_AGENTS = [\r\n    'GPTBot',\r\n    'ChatGPT-User',\r\n    'ClaudeBot',\r\n    'Claude-Web',\r\n    'anthropic-ai',\r\n    'PerplexityBot',\r\n    'OAI-SearchBot',\r\n    'GoogleExtended',\r\n];\r\n\r\nexport function ontoMiddleware(request: NextRequest) {\r\n    const userAgent = request.headers.get('user-agent') || '';\r\n    const accept = request.headers.get('accept') || '';\r\n\r\n    const isAiBot = AI_BOT_USER_AGENTS.some(bot => userAgent.includes(bot));\r\n    const isMarkdownRequested = accept.includes('text/markdown');\r\n\r\n    // If traffic is identified as an AI Bot, rewrite the URL\r\n    if (isAiBot || isMarkdownRequested) {\r\n        const url = request.nextUrl.clone();\r\n\r\n        // Ignore internal next.js requests & static assets\r\n        if (url.pathname.startsWith('/_next') || url.pathname.includes('.')) {\r\n            return NextResponse.next();\r\n        }\r\n\r\n        // Determine the corresponding payload path\r\n        let payloadPath = url.pathname;\r\n        if (payloadPath === '/' || payloadPath === '') {\r\n            payloadPath = '/index';\r\n        }\r\n\r\n        // Strip trailing slash if present\r\n        if (payloadPath.endsWith('/') && payloadPath !== '/') {\r\n            payloadPath = payloadPath.slice(0, -1);\r\n        }\r\n\r\n        url.pathname = `/.onto${payloadPath}.md`;\r\n\r\n        // Rewrite implicitly serves the target URL transparently to the client.\r\n        return NextResponse.rewrite(url);\r\n    }\r\n\r\n    return NextResponse.next();\r\n}\r\n"],"mappings":"AAAA,OAAsB,gBAAAA,MAAoB,cAE1C,IAAMC,EAAqB,CACvB,SACA,eACA,YACA,aACA,eACA,gBACA,gBACA,gBACJ,EAEO,SAASC,EAAeC,EAAsB,CACjD,IAAMC,EAAYD,EAAQ,QAAQ,IAAI,YAAY,GAAK,GACjDE,EAASF,EAAQ,QAAQ,IAAI,QAAQ,GAAK,GAE1CG,EAAUL,EAAmB,KAAKM,GAAOH,EAAU,SAASG,CAAG,CAAC,EAChEC,EAAsBH,EAAO,SAAS,eAAe,EAG3D,GAAIC,GAAWE,EAAqB,CAChC,IAAMC,EAAMN,EAAQ,QAAQ,MAAM,EAGlC,GAAIM,EAAI,SAAS,WAAW,QAAQ,GAAKA,EAAI,SAAS,SAAS,GAAG,EAC9D,OAAOT,EAAa,KAAK,EAI7B,IAAIU,EAAcD,EAAI,SACtB,OAAIC,IAAgB,KAAOA,IAAgB,MACvCA,EAAc,UAIdA,EAAY,SAAS,GAAG,GAAKA,IAAgB,MAC7CA,EAAcA,EAAY,MAAM,EAAG,EAAE,GAGzCD,EAAI,SAAW,SAASC,CAAW,MAG5BV,EAAa,QAAQS,CAAG,CACnC,CAEA,OAAOT,EAAa,KAAK,CAC7B","names":["NextResponse","AI_BOT_USER_AGENTS","ontoMiddleware","request","userAgent","accept","isAiBot","bot","isMarkdownRequested","url","payloadPath"]}

package/package.json

@@ -0,0 +1,53 @@
+{
+    "name": "@ontosdk/next",
+    "version": "1.0.0",
+    "description": "Extracts semantic Markdown from React/Next.js pages for AI Agents",
+    "main": "dist/index.js",
+    "module": "dist/index.mjs",
+    "types": "dist/index.d.ts",
+    "publishConfig": {
+        "access": "public"
+    },
+    "exports": {
+        ".": {
+            "require": "./dist/index.js",
+            "import": "./dist/index.mjs",
+            "types": "./dist/index.d.ts"
+        },
+        "./middleware": {
+            "require": "./dist/middleware.js",
+            "import": "./dist/middleware.mjs",
+            "types": "./dist/middleware.d.ts"
+        }
+    },
+    "bin": {
+        "onto-next": "dist/cli.js"
+    },
+    "scripts": {
+        "build": "tsup",
+        "dev": "tsup --watch",
+        "test": "echo \"Error: no test specified\" && exit 1"
+    },
+    "keywords": [
+        "nextjs",
+        "ai",
+        "markdown",
+        "extraction",
+        "seo"
+    ],
+    "author": "Onto",
+    "license": "MIT",
+    "dependencies": {
+        "cheerio": "^1.0.0-rc.12",
+        "glob": "^10.3.10",
+        "picocolors": "^1.0.0",
+        "turndown": "^7.1.3"
+    },
+    "devDependencies": {
+        "@types/node": "^20.11.24",
+        "@types/turndown": "^5.0.4",
+        "next": "^16.1.6",
+        "tsup": "^8.0.2",
+        "typescript": "^5.3.3"
+    }
+}

package/src/cli.ts

@@ -0,0 +1,93 @@
+#!/usr/bin/env node
+import { glob } from 'glob';
+import fs from 'fs';
+import path from 'path';
+import pc from 'picocolors';
+import { extractContent } from './extractor';
+
+async function main() {
+    console.log(pc.cyan('\n[Onto] Starting Semantic Output Generation...'));
+
+    const cwd = process.cwd();
+    const nextAppDirDir = path.join(cwd, '.next/server/app');
+    const ontoPublicDir = path.join(cwd, 'public/.onto');
+
+    if (!fs.existsSync(nextAppDirDir)) {
+        console.log(pc.yellow(`[Onto] Could not find Next.js app output at ${nextAppDirDir}`));
+        console.log(pc.yellow(`[Onto] Ensure this is run after "next build" and you are using the App Router.`));
+        return;
+    }
+
+    // Find all HTML files rendered by Next.js in the app directory
+    const files = await glob('**/*.html', { cwd: nextAppDirDir });
+
+    if (files.length === 0) {
+        console.log(pc.yellow(`[Onto] No static HTML files found to process.`));
+        return;
+    }
+
+    // Ensure output directory exists
+    if (!fs.existsSync(ontoPublicDir)) {
+        fs.mkdirSync(ontoPublicDir, { recursive: true });
+    }
+
+    let totalOriginalSize = 0;
+    let totalMarkdownSize = 0;
+    let totalFilesProcessed = 0;
+
+    for (const file of files) {
+        const inputPath = path.join(nextAppDirDir, file);
+
+        // We map file path e.g. "pricing.html" to "pricing.md", or "blog/post.html" to "blog/post.md"
+        let outputPathRelative = file.replace(/\.html$/, '.md');
+        // If it's a dynamic route page, or purely root index.html
+        const outputPath = path.join(ontoPublicDir, outputPathRelative);
+
+        try {
+            const htmlContent = fs.readFileSync(inputPath, 'utf8');
+
+            const result = extractContent(htmlContent, `/${outputPathRelative.replace(/\.md$/, '')}`);
+
+            // Ensure specific sub-directory exists (e.g., for blog/post.md)
+            const outputDir = path.dirname(outputPath);
+            if (!fs.existsSync(outputDir)) {
+                fs.mkdirSync(outputDir, { recursive: true });
+            }
+
+            fs.writeFileSync(outputPath, result.markdown, 'utf8');
+
+            totalOriginalSize += result.stats.originalHtmlSize;
+            totalMarkdownSize += result.stats.markdownSize;
+            totalFilesProcessed++;
+
+            const origKb = (result.stats.originalHtmlSize / 1024).toFixed(1);
+            const mdKb = (result.stats.markdownSize / 1024).toFixed(1);
+
+            // /index.html -> /
+            let routeName = file.replace(/\.html$/, '');
+            if (routeName === 'index') routeName = '/';
+            else routeName = `/${routeName}`;
+
+            console.log(
+                pc.green(`✓ Optimized`) +
+                pc.dim(` ${routeName} `) +
+                pc.blue(`[${origKb}KB -> ${mdKb}KB]`)
+            );
+        } catch (e: any) {
+            console.error(pc.red(`✗ Failed to process ${file}: ${e.message}`));
+        }
+    }
+
+    console.log(pc.cyan(`\n[Onto] Finished generation.`));
+    console.log(
+        pc.bold(
+            pc.magenta(`Processed ${totalFilesProcessed} pages. Total Size: ${(totalOriginalSize / 1024).toFixed(1)}KB -> ${(totalMarkdownSize / 1024).toFixed(1)}KB`)
+        )
+    );
+    console.log(pc.dim(`Edge payloads are ready at /public/.onto/*\n`));
+}
+
+main().catch(e => {
+    console.error(pc.red(`[Onto] Fatal Error: ${e.message}`));
+    process.exit(1);
+});

package/src/extractor.ts

@@ -0,0 +1,156 @@
+import * as cheerio from 'cheerio';
+import TurndownService from 'turndown';
+
+const turndownService = new TurndownService({
+    headingStyle: 'atx',
+    codeBlockStyle: 'fenced',
+});
+
+// Configure turndown to keep some layout or handle semantic tags differently if needed
+
+export interface ExtractionResult {
+    markdown: string;
+    metadata: {
+        title: string;
+        description: string;
+        jsonLd: any[];
+    };
+    stats: {
+        originalHtmlSize: number;
+        markdownSize: number;
+        tokenReductionRatio: number;
+    };
+}
+
+/**
+ * Extracts pure semantic markdown and metadata from rendered Next.js HTML strings.
+ * @param html The raw HTML string.
+ * @param sourceUrl (Optional) the URL this was generated from, to attach as metadata.
+ * @returns {ExtractionResult} The extracted payload.
+ */
+export function extractContent(html: string, sourceUrl: string = 'Generated Output'): ExtractionResult {
+    const originalSize = html.length;
+
+    const $ = cheerio.load(html);
+
+    // 1. Extract Metadata BEFORE removing structure
+    const title = $('title').text() || $('h1').first().text() || 'Untitled Page';
+    const description = $('meta[name="description"]').attr('content') || 'No description found.';
+
+    const jsonLdScripts: any[] = [];
+    $('script[type="application/ld+json"]').each((_, el) => {
+        try {
+            const raw = $(el).html() || '';
+            const parsed = JSON.parse(raw);
+            jsonLdScripts.push(parsed);
+        } catch {
+            // ignore bad json
+        }
+    });
+
+    // 2. Strip noise (React boilerplate, styles, unnecessary tags)
+    $('script, style, noscript, iframe, svg, nav, footer, meta, link, header').remove();
+
+    // Optionally remove typical Next.js hidden wrappers if they don't contain real content.
+    // Next.js uses <div id="__next"> but we mostly just want semantic content.
+
+    // 3. Find the entry point for content
+    // Prefer <main> or <article> over <body>
+    let contentHtml = '';
+    if ($('main').length > 0) {
+        contentHtml = $('main').html() || '';
+    } else if ($('article').length > 0) {
+        contentHtml = $('article').html() || '';
+    } else {
+        contentHtml = $('body').html() || '';
+    }
+
+    // 4. Convert to Markdown
+    let markdown = turndownService.turndown(contentHtml);
+
+    // 5. Optionally inject Metadata header
+    const headerLines = [
+        `# ${title}`,
+        `> ${description}`,
+        ``,
+        `**Source:** ${sourceUrl}`,
+        `**Extracted:** ${new Date().toISOString()}`,
+        ``,
+        `---`,
+        ``
+    ];
+
+    let finalMarkdown = headerLines.join('\n') + markdown;
+
+    // Add JSON-LD section if exists
+    if (jsonLdScripts.length > 0) {
+        finalMarkdown += '\n\n---\n## Structured Data (JSON-LD)\n```json\n';
+        jsonLdScripts.forEach(j => {
+            finalMarkdown += JSON.stringify(j, null, 2) + '\n';
+        });
+        finalMarkdown += '```\n';
+    }
+
+    const markdownSize = finalMarkdown.length;
+    const tokenReductionRatio = originalSize > 0 ? ((originalSize - markdownSize) / originalSize) * 100 : 0;
+
+    return {
+        markdown: finalMarkdown,
+        metadata: {
+            title,
+            description,
+            jsonLd: jsonLdScripts
+        },
+        stats: {
+            originalHtmlSize: originalSize,
+            markdownSize,
+            tokenReductionRatio
+        }
+    };
+}
+
+export async function generateStaticPayloads(nextAppDirDir: string, ontoPublicDir: string) {
+    const fs = await import('fs');
+    const path = await import('path');
+    const { glob } = await import('glob');
+
+    if (!fs.existsSync(nextAppDirDir)) {
+        return;
+    }
+
+    const files = await glob('**/*.html', { cwd: nextAppDirDir });
+    if (files.length === 0) return;
+
+    if (!fs.existsSync(ontoPublicDir)) {
+        fs.mkdirSync(ontoPublicDir, { recursive: true });
+    }
+
+    let totalFilesProcessed = 0;
+
+    for (const file of files) {
+        const inputPath = path.join(nextAppDirDir, file);
+        const outputPathRelative = file.replace(/\.html$/, '.md');
+        const outputPath = path.join(ontoPublicDir, outputPathRelative);
+
+        try {
+            const htmlContent = fs.readFileSync(inputPath, 'utf8');
+
+            let routeName = file.replace(/\.html$/, '');
+            if (routeName === 'index') routeName = '/';
+            else routeName = `/${routeName}`;
+
+            const result = extractContent(htmlContent, routeName);
+
+            const outputDir = path.dirname(outputPath);
+            if (!fs.existsSync(outputDir)) {
+                fs.mkdirSync(outputDir, { recursive: true });
+            }
+
+            fs.writeFileSync(outputPath, result.markdown, 'utf8');
+            totalFilesProcessed++;
+        } catch (e: any) {
+            console.error(`[Onto] Failed to process ${file}: ${e.message}`);
+        }
+    }
+    console.log(`[Onto] Successfully generated ${totalFilesProcessed} semantic markdown endpoints.`);
+}

package/src/index.ts

@@ -0,0 +1,3 @@
+// We cannot use Webpack plugins reliably in Next.js Turbopack due to WorkerError restrictions.
+// Users must instead run `npx onto-next` as a postbuild script.
+export { extractContent } from './extractor';

package/src/middleware.ts

@@ -0,0 +1,48 @@
+import { NextRequest, NextResponse } from 'next/server';
+
+const AI_BOT_USER_AGENTS = [
+    'GPTBot',
+    'ChatGPT-User',
+    'ClaudeBot',
+    'Claude-Web',
+    'anthropic-ai',
+    'PerplexityBot',
+    'OAI-SearchBot',
+    'GoogleExtended',
+];
+
+export function ontoMiddleware(request: NextRequest) {
+    const userAgent = request.headers.get('user-agent') || '';
+    const accept = request.headers.get('accept') || '';
+
+    const isAiBot = AI_BOT_USER_AGENTS.some(bot => userAgent.includes(bot));
+    const isMarkdownRequested = accept.includes('text/markdown');
+
+    // If traffic is identified as an AI Bot, rewrite the URL
+    if (isAiBot || isMarkdownRequested) {
+        const url = request.nextUrl.clone();
+
+        // Ignore internal next.js requests & static assets
+        if (url.pathname.startsWith('/_next') || url.pathname.includes('.')) {
+            return NextResponse.next();
+        }
+
+        // Determine the corresponding payload path
+        let payloadPath = url.pathname;
+        if (payloadPath === '/' || payloadPath === '') {
+            payloadPath = '/index';
+        }
+
+        // Strip trailing slash if present
+        if (payloadPath.endsWith('/') && payloadPath !== '/') {
+            payloadPath = payloadPath.slice(0, -1);
+        }
+
+        url.pathname = `/.onto${payloadPath}.md`;
+
+        // Rewrite implicitly serves the target URL transparently to the client.
+        return NextResponse.rewrite(url);
+    }
+
+    return NextResponse.next();
+}

package/tsconfig.json

@@ -0,0 +1,15 @@
+{
+    "compilerOptions": {
+        "target": "es2022",
+        "module": "esnext",
+        "moduleResolution": "node",
+        "esModuleInterop": true,
+        "strict": true,
+        "skipLibCheck": true,
+        "forceConsistentCasingInFileNames": true,
+        "outDir": "dist"
+    },
+    "include": [
+        "src/**/*"
+    ]
+}

package/tsup.config.ts

@@ -0,0 +1,14 @@
+import { defineConfig } from 'tsup';
+
+export default defineConfig({
+    entry: ['src/index.ts', 'src/cli.ts', 'src/middleware.ts'],
+    format: ['cjs', 'esm'],
+    dts: true,
+    splitting: false,
+    sourcemap: true,
+    clean: true,
+    bundle: true,
+    outDir: 'dist',
+    minify: true,
+    external: ['next'],
+});