@peam-ai/next 0.1.3 → 0.1.5

package/dist/peam.adapter.js

@@ -1,10 +1,10 @@
 'use strict';
 
 var fs = require('fs');
-var path = require('path');
 var logger = require('peam/logger');
 var parser = require('peam/parser');
 var search = require('peam/search');
+var search$1 = require('@peam-ai/search');
 
 // src/adapter.ts
 var log = logger.loggers.adapter;
@@ -118,42 +118,32 @@ function createPeamAdapter(config) {
           log.error("Error processing", pathname, error);
         }
       }
-      const outputPath = path.join(projectDir, config.outputDir);
-      fs.mkdirSync(outputPath, { recursive: true });
       log.debug("Creating search index...");
       const searchIndexData = await search.buildSearchIndex(pages);
-      const generatedPath = path.join(outputPath, "generated");
-      fs.mkdirSync(generatedPath, { recursive: true });
-      const searchIndexFile = path.join(generatedPath, config.indexFilename);
-      fs.writeFileSync(searchIndexFile, JSON.stringify(searchIndexData));
-      const indexJsContent = `// Auto-generated by Peam - DO NOT EDIT THIS FILE
-import index from "./generated/${config.indexFilename}";
-export default index;
-`;
-      const indexJsFile = path.join(outputPath, "index.js");
-      fs.writeFileSync(indexJsFile, indexJsContent);
-      log.debug("Saved search index to:", searchIndexFile);
+      await config.searchIndexExporter.export(searchIndexData);
+      log.debug("Saved search index via exporter");
       log.debug("Extraction complete with total pages:", pages.length);
     }
   };
 }
-
-// src/config.ts
-var defaultConfig = {
-  outputDir: ".peam",
-  indexFilename: "index.json",
-  respectRobotsTxt: true,
-  robotsTxtPath: void 0,
-  exclude: []
-};
 var getConfig = () => {
-  return {
-    outputDir: process.env.PEAM_OUTPUT_DIR || defaultConfig.outputDir,
-    indexFilename: process.env.PEAM_INDEX_FILENAME || defaultConfig.indexFilename,
-    respectRobotsTxt: process.env.PEAM_RESPECT_ROBOTS_TXT !== void 0 ? process.env.PEAM_RESPECT_ROBOTS_TXT === "true" : defaultConfig.respectRobotsTxt,
-    robotsTxtPath: process.env.PEAM_ROBOTS_TXT_PATH || defaultConfig.robotsTxtPath,
-    exclude: process.env.PEAM_EXCLUDE ? JSON.parse(process.env.PEAM_EXCLUDE) : defaultConfig.exclude
+  if (!process.env.PEAM_SEARCH_EXPORTER_TYPE || !process.env.PEAM_SEARCH_EXPORTER_CONFIG) {
+    throw new Error(
+      "Peam configuration not found. Make sure withPeam() is properly configured in your next.config file."
+    );
+  }
+  const searchExporterConfig = {
+    type: process.env.PEAM_SEARCH_EXPORTER_TYPE,
+    config: JSON.parse(process.env.PEAM_SEARCH_EXPORTER_CONFIG)
+  };
+  const resolvedConfig = {
+    searchExporter: searchExporterConfig,
+    searchIndexExporter: search$1.createExporterFromConfig(searchExporterConfig),
+    respectRobotsTxt: process.env.PEAM_RESPECT_ROBOTS_TXT === "true",
+    robotsTxtPath: process.env.PEAM_ROBOTS_TXT_PATH || void 0,
+    exclude: process.env.PEAM_EXCLUDE ? JSON.parse(process.env.PEAM_EXCLUDE) : []
   };
+  return resolvedConfig;
 };
 
 // src/peam.adapter.ts

package/dist/peam.adapter.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/adapter.ts","../src/config.ts","../src/peam.adapter.ts"],"names":["loggers","readFileSync","baseLoadRobotsTxt","createRobotsParser","shouldIncludePath","parseHTML","join","mkdirSync","buildSearchIndex","writeFileSync"],"mappings":";;;;;;;;;AAeA,IAAM,MAAMA,cAAA,CAAQ,OAAA;AAiBpB,SAAS,2BAA2B,SAAA,EAA2C;AAhC/E,EAAA,IAAA,EAAA;AAiCE,EAAA,IAAI;AACF,IAAA,IAAI,SAAA,CAAU,aAAa,aAAA,EAAe;AACxC,MAAA,OAAO,IAAA;AAAA,IACT;AAEA,IAAA,IAAA,CAAI,EAAA,GAAA,SAAA,CAAU,QAAA,KAAV,IAAA,GAAA,KAAA,CAAA,GAAA,EAAA,CAAoB,QAAA,EAAU;AAChC,MAAA,MAAM,OAAA,GAAUC,eAAA,CAAa,SAAA,CAAU,QAAA,CAAS,UAAU,OAAO,CAAA;AACjE,MAAA,OAAO,OAAA;AAAA,IACT;AAAA,EACF,SAAS,KAAA,EAAO;AACd,IAAA,GAAA,CAAI,KAAA,CAAM,2CAA2C,KAAK,CAAA;AAAA,EAC5D;AAEA,EAAA,OAAO,IAAA;AACT;AAEA,SAAS,aAAA,CACP,UAAA,EACA,UAAA,EACA,aAAA,EACwB;AACxB,EAAA,IAAI;AACF,IAAA,IAAI,aAAA,GAA+B,IAAA;AACnC,IAAA,IAAI,SAAA,GAA2B,IAAA;AAE/B,IAAA,IAAI,UAAA,IAAc,UAAA,CAAW,MAAA,GAAS,CAAA,EAAG;AACvC,MAAA,KAAA,MAAW,aAAa,UAAA,EAAY;AAClC,QAAA,MAAM,OAAA,GAAU,2BAA2B,SAAS,CAAA;AACpD,QAAA,IAAI,OAAA,EAAS;AACX,UAAA,GAAA,CAAI,MAAM,0CAA0C,CAAA;AACpD,UAAA,aAAA,GAAgB,OAAA;AAChB,UAAA,SAAA,GAAY,SAAA,CAAU,QAAA;AACtB,UAAA;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,IAAA,IAAI,CAAC,aAAA,EAAe;AAClB,MAAA,MAAM,WAAA,GAAc,CAAC,mBAAA,EAAqB,gBAAA,EAAkB,oBAAoB,CAAA;AAChF,MAAA,MAAM,MAAA,GAASC,oBAAA,CAAkB,UAAA,EAAY,WAAA,EAAa,aAAa,CAAA;AACvE,MAAA,IAAI,MAAA,EAAQ;AACV,QAAA,GAAA,CAAI,KAAA,CAAM,yBAAA,EAA2B,MAAA,CAAO,IAAI,CAAA;AAChD,QAAA,OAAO,MAAA;AAAA,MACT;AACA,MAAA,OAAO,IAAA;AAAA,IACT;AAEA,IAAA,OAAO;AAAA,MACL,MAAA,EAAQC,0BAAmB,aAAa,CAAA;AAAA,MACxC,MAAM,SAAA,IAAa;AAAA,KACrB;AAAA,EACF,SAAS,KAAA,EAAO;AACd,IAAA,GAAA,CAAI,KAAA,CAAM,6BAA6B,KAAK,CAAA;AAC5C,IAAA,OAAO,IAAA;AAAA,EACT;AACF;AAEO,SAAS,kBAAkB,MAAA,EAAgD;AAChF,EAAA,OAAO;AAAA,IACL,IAAA,EAAM,cAAA;AAAA,IAEN,MAAM,gBAAgB,GAAA,EAAK;AA9F/B,MAAA,IAAA,EAAA,EAAA,EAAA;AA+FM,MAAA,GAAA,CAAI,MAAM,qCAAqC,CAAA;AAE/C,MAAA,MAAM,UAAU,GAAA,CAAI,OAAA;AACpB,MAAA,IAAI,UAAA;AAEJ,MAAA,IAAI,KAAA,CAAM,OAAA,CAAQ,OAAO,CAAA,EAAG;AAC1B,QAAA,UAAA,GAAa,QAAQ,MAAA,CAAO,CAAC,MAAA,KAA2B,MAAA,CAAO,SAAS,WAAW,CAAA;AAAA,MACrF,CAAA,MAAO;AACL,QAAA,UAAA,GAAa,OAAA,CAAQ,cAAc,EAAC;AAAA,MACtC;AAEA,MAAA,GAAA,CAAI,KAAA,CAAM,mBAAA,EAAqB,UAAA,CAAW,MAAM,CAAA;AAEhD,MAAA,MAAM,UAAA,GAAa,GAAA,CAAI,UAAA,IAAc,OAAA,CAAQ,GAAA,EAAI;AAEjD,MAAA,MAAM,YAAA,GAAe,OAAO,gBAAA,GAAmB,aAAA,CAAc,YAAY,UAAA,EAAY,MAAA,CAAO,aAAa,CAAA,GAAI,IAAA;AAE7G,MAAA,IAAI,YAAA,EAAc;AAChB,QAAA,GAAA,CAAI,KAAA,CAAM,wBAAA,EAA0B,YAAA,CAAa,IAAI,CAAA;AAAA,MACvD;AAEA,MAAA,MAAM,QAMD,EAAC;AAEN,MAAA,KAAA,MAAW,aAAa,UAAA,EAAY;AAClC,QAAA,MAAM,WAAW,SAAA,CAAU,QAAA;AAC3B,QAAA,IAAI,gBAAA,GAAA,CAAmB,EAAA,GAAA,SAAA,CAAU,QAAA,KAAV,IAAA,GAAA,MAAA,GAAA,EAAA,CAAoB,QAAA;AAE3C,QAAA,IAAI,CAAC,gBAAA,EAAkB;AACrB,UAAA;AAAA,QACF;AAGA,QAAA,IAAI,gBAAA,IAAA,IAAA,GAAA,MAAA,GAAA,gBAAA,CAAkB,SAAS,QAAA,CAAA,EAAW;AACxC,UAAA,gBAAA,GAAmB,gBAAA,CAAiB,OAAA,CAAQ,QAAA,EAAU,aAAa,CAAA;AAAA,QACrE;AAEA,QAAA,MAAM,YAAA,GAAeC,wBAAA;AAAA,UACnB,QAAA;AAAA,UAAA,CACA,EAAA,GAAA,YAAA,IAAA,IAAA,GAAA,MAAA,GAAA,YAAA,CAAc,WAAd,IAAA,GAAA,EAAA,GAAwB,IAAA;AAAA,UACxB,MAAA,CAAO,OAAA;AAAA,UACP,MAAA,CAAO;AAAA,SACT;AAEA,QAAA,IAAI,CAAC,aAAa,QAAA,EAAU;AAC1B,UAAA,IAAI,YAAA,CAAa,WAAW,YAAA,EAAc;AACxC,YAAA,GAAA,CAAI,KAAA,CAAM,gCAAgC,QAAQ,CAAA;AAAA,UACpD,CAAA,MAAA,IAAW,YAAA,CAAa,MAAA,KAAW,iBAAA,EAAmB;AACpD,YAAA,GAAA,CAAI,KAAA,CAAM,kCAAkC,QAAQ,CAAA;AAAA,UACtD;AACA,UAAA;AAAA,QACF;AAEA,QAAA,IAAI;AACF,UAAA,GAAA,CAAI,KAAA,CAAM,sBAAsB,gBAAgB,CAAA;AAEhD,UAAA,MAAM,IAAA,GAAOH,eAAA,CAAa,gBAAA,EAAkB,OAAO,CAAA;AACnD,UAAA,MAAM,cAAA,GAAiBI,iBAAU,IAAI,CAAA;AAErC,UAAA,IAAI,CAAC,cAAA,EAAgB;AACnB,YAAA,GAAA,CAAI,IAAA,CAAK,6BAA6B,QAAQ,CAAA;AAC9C,YAAA;AAAA,UACF;AAEA,UAAA,GAAA,CAAI,KAAA,CAAM,uCAAuC,QAAQ,CAAA;AACzD,UAAA,KAAA,CAAM,IAAA,CAAK;AAAA,YACT,IAAA,EAAM,QAAA;AAAA,YACN,QAAA,EAAU,gBAAA,CAAiB,OAAA,CAAQ,UAAA,GAAa,KAAK,EAAE,CAAA;AAAA,YACvD,cAAA;AAAA,YACA,IAAA,EAAM;AAAA,WACP,CAAA;AAAA,QACH,SAAS,KAAA,EAAO;AACd,UAAA,GAAA,CAAI,KAAA,CAAM,kBAAA,EAAoB,QAAA,EAAU,KAAK,CAAA;AAAA,QAC/C;AAAA,MACF;AAEA,MAAA,MAAM,UAAA,GAAaC,SAAA,CAAK,UAAA,EAAY,MAAA,CAAO,SAAS,CAAA;AACpD,MAAAC,YAAA,CAAU,UAAA,EAAY,EAAE,SAAA,EAAW,IAAA,EAAM,CAAA;AAEzC,MAAA,GAAA,CAAI,MAAM,0BAA0B,CAAA;AACpC,MAAA,MAAM,eAAA,GAAkB,MAAMC,uBAAA,CAAiB,KAAK,CAAA;AAEpD,MAAA,MAAM,aAAA,GAAgBF,SAAA,CAAK,UAAA,EAAY,WAAW,CAAA;AAClD,MAAAC,YAAA,CAAU,aAAA,EAAe,EAAE,SAAA,EAAW,IAAA,EAAM,CAAA;AAE5C,MAAA,MAAM,eAAA,GAAkBD,SAAA,CAAK,aAAA,EAAe,MAAA,CAAO,aAAa,CAAA;AAChE,MAAAG,gBAAA,CAAc,eAAA,EAAiB,IAAA,CAAK,SAAA,CAAU,eAAe,CAAC,CAAA;AAE9D,MAAA,MAAM,cAAA,GAAiB,CAAA;AAAA,+BAAA,EACI,OAAO,aAAa,CAAA;AAAA;AAAA,CAAA;AAG/C,MAAA,MAAM,WAAA,GAAcH,SAAA,CAAK,UAAA,EAAY,UAAU,CAAA;AAC/C,MAAAG,gBAAA,CAAc,aAAa,cAAc,CAAA;AAEzC,MAAA,GAAA,CAAI,KAAA,CAAM,0BAA0B,eAAe,CAAA;AACnD,MAAA,GAAA,CAAI,KAAA,CAAM,uCAAA,EAAyC,KAAA,CAAM,MAAM,CAAA;AAAA,IACjE;AAAA,GACF;AACF;;;ACjKO,IAAM,aAAA,GAAgB;AAAA,EAC3B,SAAA,EAAW,OAAA;AAAA,EACX,aAAA,EAAe,YAAA;AAAA,EACf,gBAAA,EAAkB,IAAA;AAAA,EAClB,aAAA,EAAe,MAAA;AAAA,EACf,SAAS;AACX,CAAA;AAEO,IAAM,YAAY,MAAiC;AACxD,EAAA,OAAO;AAAA,IACL,SAAA,EAAW,OAAA,CAAQ,GAAA,CAAI,eAAA,IAAmB,aAAA,CAAc,SAAA;AAAA,IACxD,aAAA,EAAe,OAAA,CAAQ,GAAA,CAAI,mBAAA,IAAuB,aAAA,CAAc,aAAA;AAAA,IAChE,gBAAA,EACE,QAAQ,GAAA,CAAI,uBAAA,KAA4B,SACpC,OAAA,CAAQ,GAAA,CAAI,uBAAA,KAA4B,MAAA,GACxC,aAAA,CAAc,gBAAA;AAAA,IACpB,aAAA,EAAe,OAAA,CAAQ,GAAA,CAAI,oBAAA,IAAwB,aAAA,CAAc,aAAA;AAAA,IACjE,OAAA,EAAS,OAAA,CAAQ,GAAA,CAAI,YAAA,GAAe,IAAA,CAAK,MAAM,OAAA,CAAQ,GAAA,CAAI,YAAY,CAAA,GAAI,aAAA,CAAc;AAAA,GAC3F;AACF,CAAA;;;ACtDA,IAAO,oBAAA,GAAQ,iBAAA,CAAkB,SAAA,EAAW","file":"peam.adapter.js","sourcesContent":["import { mkdirSync, readFileSync, writeFileSync } from 'fs';\nimport type { NextAdapter } from 'next';\nimport { join } from 'path';\nimport { loggers } from 'peam/logger';\nimport {\n  loadRobotsTxt as baseLoadRobotsTxt,\n  createRobotsParser,\n  parseHTML,\n  shouldIncludePath,\n  type RobotsTxtResult,\n  type StructuredPage,\n} from 'peam/parser';\nimport { buildSearchIndex } from 'peam/search';\nimport { type ResolvedPeamAdapterConfig } from './config';\n\nconst log = loggers.adapter;\n\ninterface PrerenderOutput {\n  pathname: string;\n  fallback?: {\n    filePath: string;\n  };\n}\n\ninterface NextJS15Output extends PrerenderOutput {\n  type: string;\n}\n\ninterface NextJS16Outputs {\n  prerenders: Array<PrerenderOutput>;\n}\n\nfunction extractRobotsFromPrerender(prerender: PrerenderOutput): string | null {\n  try {\n    if (prerender.pathname !== '/robots.txt') {\n      return null;\n    }\n\n    if (prerender.fallback?.filePath) {\n      const content = readFileSync(prerender.fallback.filePath, 'utf-8');\n      return content;\n    }\n  } catch (error) {\n    log.error('Error extracting robots from prerender:', error);\n  }\n\n  return null;\n}\n\nfunction loadRobotsTxt(\n  projectDir: string,\n  prerenders: PrerenderOutput[],\n  robotsTxtPath?: string\n): RobotsTxtResult | null {\n  try {\n    let robotsContent: string | null = null;\n    let foundPath: string | null = null;\n\n    if (prerenders && prerenders.length > 0) {\n      for (const prerender of prerenders) {\n        const content = extractRobotsFromPrerender(prerender);\n        if (content) {\n          log.debug('Found dynamic robots.txt from prerenders');\n          robotsContent = content;\n          foundPath = prerender.pathname;\n          break;\n        }\n      }\n    }\n\n    if (!robotsContent) {\n      const searchPaths = ['public/robots.txt', 'app/robots.txt', 'src/app/robots.txt'];\n      const result = baseLoadRobotsTxt(projectDir, searchPaths, robotsTxtPath);\n      if (result) {\n        log.debug('Loaded robots.txt from:', result.path);\n        return result;\n      }\n      return null;\n    }\n\n    return {\n      parser: createRobotsParser(robotsContent),\n      path: foundPath || '',\n    };\n  } catch (error) {\n    log.error('Error loading robots.txt:', error);\n    return null;\n  }\n}\n\nexport function createPeamAdapter(config: ResolvedPeamAdapterConfig): NextAdapter {\n  return {\n    name: 'peam-adapter',\n\n    async onBuildComplete(ctx) {\n      log.debug('Extracting page content via adapter');\n\n      const outputs = ctx.outputs as NextJS15Output[] | NextJS16Outputs;\n      let prerenders: PrerenderOutput[];\n\n      if (Array.isArray(outputs)) {\n        prerenders = outputs.filter((output: NextJS15Output) => output.type === 'PRERENDER');\n      } else {\n        prerenders = outputs.prerenders || [];\n      }\n\n      log.debug('Total prerenders:', prerenders.length);\n\n      const projectDir = ctx.projectDir || process.cwd();\n\n      const robotsResult = config.respectRobotsTxt ? loadRobotsTxt(projectDir, prerenders, config.robotsTxtPath) : null;\n\n      if (robotsResult) {\n        log.debug('Using robots.txt from:', robotsResult.path);\n      }\n\n      const pages: Array<{\n        path: string;\n        htmlFile: string;\n        structuredPage: StructuredPage;\n        type: string;\n        runtime?: string;\n      }> = [];\n\n      for (const prerender of prerenders) {\n        const pathname = prerender.pathname;\n        let fallbackFilePath = prerender.fallback?.filePath;\n\n        if (!fallbackFilePath) {\n          continue;\n        }\n\n        // Fix for Next.js 15\n        if (fallbackFilePath?.endsWith('/.html')) {\n          fallbackFilePath = fallbackFilePath.replace('/.html', '/index.html');\n        }\n\n        const filterResult = shouldIncludePath(\n          pathname,\n          robotsResult?.parser ?? null,\n          config.exclude,\n          config.respectRobotsTxt\n        );\n\n        if (!filterResult.included) {\n          if (filterResult.reason === 'robots-txt') {\n            log.debug('Path excluded by robots.txt:', pathname);\n          } else if (filterResult.reason === 'exclude-pattern') {\n            log.debug('Path excluded by user pattern:', pathname);\n          }\n          continue;\n        }\n\n        try {\n          log.debug('Reading HTML from:', fallbackFilePath);\n\n          const html = readFileSync(fallbackFilePath, 'utf-8');\n          const structuredPage = parseHTML(html);\n\n          if (!structuredPage) {\n            log.warn('No content extracted from', pathname);\n            continue;\n          }\n\n          log.debug('Successfully extracted content from', pathname);\n          pages.push({\n            path: pathname,\n            htmlFile: fallbackFilePath.replace(projectDir + '/', ''),\n            structuredPage,\n            type: 'page',\n          });\n        } catch (error) {\n          log.error('Error processing', pathname, error);\n        }\n      }\n\n      const outputPath = join(projectDir, config.outputDir);\n      mkdirSync(outputPath, { recursive: true });\n\n      log.debug('Creating search index...');\n      const searchIndexData = await buildSearchIndex(pages);\n\n      const generatedPath = join(outputPath, 'generated');\n      mkdirSync(generatedPath, { recursive: true });\n\n      const searchIndexFile = join(generatedPath, config.indexFilename);\n      writeFileSync(searchIndexFile, JSON.stringify(searchIndexData));\n\n      const indexJsContent = `// Auto-generated by Peam - DO NOT EDIT THIS FILE\nimport index from \"./generated/${config.indexFilename}\";\nexport default index;\n`;\n      const indexJsFile = join(outputPath, 'index.js');\n      writeFileSync(indexJsFile, indexJsContent);\n\n      log.debug('Saved search index to:', searchIndexFile);\n      log.debug('Extraction complete with total pages:', pages.length);\n    },\n  };\n}\n","import { NextConfig } from 'next';\n\nexport interface PeamAdapterConfig {\n  /**\n   * Directory where the search index and pages.json will be output\n   * @default '.peam'\n   */\n  outputDir?: string;\n  /**\n   * Filename for the search index\n   * @default 'index.json'\n   */\n  indexFilename?: string;\n  /**\n   * Whether to respect robots.txt rules when indexing pages\n   * @default true\n   */\n  respectRobotsTxt?: boolean;\n  /**\n   * Path to a custom robots.txt file relative to the project root\n   * If not specified, the adapter will look for static or dynamic robots.txt files in standard locations\n   * @example 'custom/robots.txt' or 'config/production-robots.txt'\n   * @default undefined\n   */\n  robotsTxtPath?: string;\n  /**\n   * Array of wildcard patterns to exclude from indexing\n   * Supports * (matches any characters except /), ** (matches any characters including /), and ? (single character)\n   * @example ['/admin/**', '/api/*', '/private-*']\n   * @default []\n   */\n  exclude?: string[];\n}\n\nexport type ResolvedPeamAdapterConfig = Required<Omit<PeamAdapterConfig, 'robotsTxtPath'>> & {\n  robotsTxtPath?: string;\n};\n\nexport const defaultConfig = {\n  outputDir: '.peam',\n  indexFilename: 'index.json',\n  respectRobotsTxt: true,\n  robotsTxtPath: undefined,\n  exclude: [],\n} satisfies PeamAdapterConfig;\n\nexport const getConfig = (): ResolvedPeamAdapterConfig => {\n  return {\n    outputDir: process.env.PEAM_OUTPUT_DIR || defaultConfig.outputDir,\n    indexFilename: process.env.PEAM_INDEX_FILENAME || defaultConfig.indexFilename,\n    respectRobotsTxt:\n      process.env.PEAM_RESPECT_ROBOTS_TXT !== undefined\n        ? process.env.PEAM_RESPECT_ROBOTS_TXT === 'true'\n        : defaultConfig.respectRobotsTxt,\n    robotsTxtPath: process.env.PEAM_ROBOTS_TXT_PATH || defaultConfig.robotsTxtPath,\n    exclude: process.env.PEAM_EXCLUDE ? JSON.parse(process.env.PEAM_EXCLUDE) : defaultConfig.exclude,\n  };\n};\n\nexport function setNextConfig(nextConfig: NextConfig, peamConfig: PeamAdapterConfig): void {\n  const envVars = {\n    PEAM_OUTPUT_DIR: peamConfig.outputDir,\n    PEAM_INDEX_FILENAME: peamConfig.indexFilename,\n    PEAM_RESPECT_ROBOTS_TXT: String(peamConfig.respectRobotsTxt),\n    PEAM_EXCLUDE: JSON.stringify(peamConfig.exclude),\n    PEAM_ROBOTS_TXT_PATH: '',\n  };\n\n  if (peamConfig.robotsTxtPath) {\n    envVars.PEAM_ROBOTS_TXT_PATH = String(peamConfig.robotsTxtPath);\n  }\n\n  // Set build time vars\n  Object.assign(process.env, envVars);\n\n  // Set runtime vars\n  nextConfig.env = {\n    ...nextConfig.env,\n    ...envVars,\n  };\n}\n","import { createPeamAdapter } from './adapter';\nimport { getConfig } from './config';\n\nexport default createPeamAdapter(getConfig());\n"]}
+{"version":3,"sources":["../src/adapter.ts","../src/config.ts","../src/peam.adapter.ts"],"names":["loggers","readFileSync","baseLoadRobotsTxt","createRobotsParser","shouldIncludePath","parseHTML","buildSearchIndex","createExporterFromConfig"],"mappings":";;;;;;;;;AAcA,IAAM,MAAMA,cAAA,CAAQ,OAAA;AAiBpB,SAAS,2BAA2B,SAAA,EAA2C;AA/B/E,EAAA,IAAA,EAAA;AAgCE,EAAA,IAAI;AACF,IAAA,IAAI,SAAA,CAAU,aAAa,aAAA,EAAe;AACxC,MAAA,OAAO,IAAA;AAAA,IACT;AAEA,IAAA,IAAA,CAAI,EAAA,GAAA,SAAA,CAAU,QAAA,KAAV,IAAA,GAAA,KAAA,CAAA,GAAA,EAAA,CAAoB,QAAA,EAAU;AAChC,MAAA,MAAM,OAAA,GAAUC,eAAA,CAAa,SAAA,CAAU,QAAA,CAAS,UAAU,OAAO,CAAA;AACjE,MAAA,OAAO,OAAA;AAAA,IACT;AAAA,EACF,SAAS,KAAA,EAAO;AACd,IAAA,GAAA,CAAI,KAAA,CAAM,2CAA2C,KAAK,CAAA;AAAA,EAC5D;AAEA,EAAA,OAAO,IAAA;AACT;AAEA,SAAS,aAAA,CACP,UAAA,EACA,UAAA,EACA,aAAA,EACwB;AACxB,EAAA,IAAI;AACF,IAAA,IAAI,aAAA,GAA+B,IAAA;AACnC,IAAA,IAAI,SAAA,GAA2B,IAAA;AAE/B,IAAA,IAAI,UAAA,IAAc,UAAA,CAAW,MAAA,GAAS,CAAA,EAAG;AACvC,MAAA,KAAA,MAAW,aAAa,UAAA,EAAY;AAClC,QAAA,MAAM,OAAA,GAAU,2BAA2B,SAAS,CAAA;AACpD,QAAA,IAAI,OAAA,EAAS;AACX,UAAA,GAAA,CAAI,MAAM,0CAA0C,CAAA;AACpD,UAAA,aAAA,GAAgB,OAAA;AAChB,UAAA,SAAA,GAAY,SAAA,CAAU,QAAA;AACtB,UAAA;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,IAAA,IAAI,CAAC,aAAA,EAAe;AAClB,MAAA,MAAM,WAAA,GAAc,CAAC,mBAAA,EAAqB,gBAAA,EAAkB,oBAAoB,CAAA;AAChF,MAAA,MAAM,MAAA,GAASC,oBAAA,CAAkB,UAAA,EAAY,WAAA,EAAa,aAAa,CAAA;AACvE,MAAA,IAAI,MAAA,EAAQ;AACV,QAAA,GAAA,CAAI,KAAA,CAAM,yBAAA,EAA2B,MAAA,CAAO,IAAI,CAAA;AAChD,QAAA,OAAO,MAAA;AAAA,MACT;AACA,MAAA,OAAO,IAAA;AAAA,IACT;AAEA,IAAA,OAAO;AAAA,MACL,MAAA,EAAQC,0BAAmB,aAAa,CAAA;AAAA,MACxC,MAAM,SAAA,IAAa;AAAA,KACrB;AAAA,EACF,SAAS,KAAA,EAAO;AACd,IAAA,GAAA,CAAI,KAAA,CAAM,6BAA6B,KAAK,CAAA;AAC5C,IAAA,OAAO,IAAA;AAAA,EACT;AACF;AAEO,SAAS,kBAAkB,MAAA,EAAgD;AAChF,EAAA,OAAO;AAAA,IACL,IAAA,EAAM,cAAA;AAAA,IAEN,MAAM,gBAAgB,GAAA,EAAK;AA7F/B,MAAA,IAAA,EAAA,EAAA,EAAA;AA8FM,MAAA,GAAA,CAAI,MAAM,qCAAqC,CAAA;AAE/C,MAAA,MAAM,UAAU,GAAA,CAAI,OAAA;AACpB,MAAA,IAAI,UAAA;AAEJ,MAAA,IAAI,KAAA,CAAM,OAAA,CAAQ,OAAO,CAAA,EAAG;AAC1B,QAAA,UAAA,GAAa,QAAQ,MAAA,CAAO,CAAC,MAAA,KAA2B,MAAA,CAAO,SAAS,WAAW,CAAA;AAAA,MACrF,CAAA,MAAO;AACL,QAAA,UAAA,GAAa,OAAA,CAAQ,cAAc,EAAC;AAAA,MACtC;AAEA,MAAA,GAAA,CAAI,KAAA,CAAM,mBAAA,EAAqB,UAAA,CAAW,MAAM,CAAA;AAEhD,MAAA,MAAM,UAAA,GAAa,GAAA,CAAI,UAAA,IAAc,OAAA,CAAQ,GAAA,EAAI;AAEjD,MAAA,MAAM,YAAA,GAAe,OAAO,gBAAA,GAAmB,aAAA,CAAc,YAAY,UAAA,EAAY,MAAA,CAAO,aAAa,CAAA,GAAI,IAAA;AAE7G,MAAA,IAAI,YAAA,EAAc;AAChB,QAAA,GAAA,CAAI,KAAA,CAAM,wBAAA,EAA0B,YAAA,CAAa,IAAI,CAAA;AAAA,MACvD;AAEA,MAAA,MAAM,QAMD,EAAC;AAEN,MAAA,KAAA,MAAW,aAAa,UAAA,EAAY;AAClC,QAAA,MAAM,WAAW,SAAA,CAAU,QAAA;AAC3B,QAAA,IAAI,gBAAA,GAAA,CAAmB,EAAA,GAAA,SAAA,CAAU,QAAA,KAAV,IAAA,GAAA,MAAA,GAAA,EAAA,CAAoB,QAAA;AAE3C,QAAA,IAAI,CAAC,gBAAA,EAAkB;AACrB,UAAA;AAAA,QACF;AAGA,QAAA,IAAI,gBAAA,IAAA,IAAA,GAAA,MAAA,GAAA,gBAAA,CAAkB,SAAS,QAAA,CAAA,EAAW;AACxC,UAAA,gBAAA,GAAmB,gBAAA,CAAiB,OAAA,CAAQ,QAAA,EAAU,aAAa,CAAA;AAAA,QACrE;AAEA,QAAA,MAAM,YAAA,GAAeC,wBAAA;AAAA,UACnB,QAAA;AAAA,UAAA,CACA,EAAA,GAAA,YAAA,IAAA,IAAA,GAAA,MAAA,GAAA,YAAA,CAAc,WAAd,IAAA,GAAA,EAAA,GAAwB,IAAA;AAAA,UACxB,MAAA,CAAO,OAAA;AAAA,UACP,MAAA,CAAO;AAAA,SACT;AAEA,QAAA,IAAI,CAAC,aAAa,QAAA,EAAU;AAC1B,UAAA,IAAI,YAAA,CAAa,WAAW,YAAA,EAAc;AACxC,YAAA,GAAA,CAAI,KAAA,CAAM,gCAAgC,QAAQ,CAAA;AAAA,UACpD,CAAA,MAAA,IAAW,YAAA,CAAa,MAAA,KAAW,iBAAA,EAAmB;AACpD,YAAA,GAAA,CAAI,KAAA,CAAM,kCAAkC,QAAQ,CAAA;AAAA,UACtD;AACA,UAAA;AAAA,QACF;AAEA,QAAA,IAAI;AACF,UAAA,GAAA,CAAI,KAAA,CAAM,sBAAsB,gBAAgB,CAAA;AAEhD,UAAA,MAAM,IAAA,GAAOH,eAAA,CAAa,gBAAA,EAAkB,OAAO,CAAA;AACnD,UAAA,MAAM,cAAA,GAAiBI,iBAAU,IAAI,CAAA;AAErC,UAAA,IAAI,CAAC,cAAA,EAAgB;AACnB,YAAA,GAAA,CAAI,IAAA,CAAK,6BAA6B,QAAQ,CAAA;AAC9C,YAAA;AAAA,UACF;AAEA,UAAA,GAAA,CAAI,KAAA,CAAM,uCAAuC,QAAQ,CAAA;AACzD,UAAA,KAAA,CAAM,IAAA,CAAK;AAAA,YACT,IAAA,EAAM,QAAA;AAAA,YACN,QAAA,EAAU,gBAAA,CAAiB,OAAA,CAAQ,UAAA,GAAa,KAAK,EAAE,CAAA;AAAA,YACvD,cAAA;AAAA,YACA,IAAA,EAAM;AAAA,WACP,CAAA;AAAA,QACH,SAAS,KAAA,EAAO;AACd,UAAA,GAAA,CAAI,KAAA,CAAM,kBAAA,EAAoB,QAAA,EAAU,KAAK,CAAA;AAAA,QAC/C;AAAA,MACF;AAEA,MAAA,GAAA,CAAI,MAAM,0BAA0B,CAAA;AACpC,MAAA,MAAM,eAAA,GAAkB,MAAMC,uBAAA,CAAiB,KAAK,CAAA;AAGpD,MAAA,MAAM,MAAA,CAAO,mBAAA,CAAoB,MAAA,CAAO,eAAe,CAAA;AAEvD,MAAA,GAAA,CAAI,MAAM,iCAAiC,CAAA;AAC3C,MAAA,GAAA,CAAI,KAAA,CAAM,uCAAA,EAAyC,KAAA,CAAM,MAAM,CAAA;AAAA,IACjE;AAAA,GACF;AACF;ACpHO,IAAM,YAAY,MAAiC;AACxD,EAAA,IAAI,CAAC,OAAA,CAAQ,GAAA,CAAI,6BAA6B,CAAC,OAAA,CAAQ,IAAI,2BAAA,EAA6B;AACtF,IAAA,MAAM,IAAI,KAAA;AAAA,MACR;AAAA,KACF;AAAA,EACF;AAEA,EAAA,MAAM,oBAAA,GAA6C;AAAA,IACjD,IAAA,EAAM,QAAQ,GAAA,CAAI,yBAAA;AAAA,IAClB,MAAA,EAAQ,IAAA,CAAK,KAAA,CAAM,OAAA,CAAQ,IAAI,2BAA2B;AAAA,GAC5D;AAEA,EAAA,MAAM,cAAA,GAAiB;AAAA,IACrB,cAAA,EAAgB,oBAAA;AAAA,IAChB,mBAAA,EAAqBC,kCAAyB,oBAAoB,CAAA;AAAA,IAClE,gBAAA,EAAkB,OAAA,CAAQ,GAAA,CAAI,uBAAA,KAA4B,MAAA;AAAA,IAC1D,aAAA,EAAe,OAAA,CAAQ,GAAA,CAAI,oBAAA,IAAwB,MAAA;AAAA,IACnD,OAAA,EAAS,OAAA,CAAQ,GAAA,CAAI,YAAA,GAAe,IAAA,CAAK,MAAM,OAAA,CAAQ,GAAA,CAAI,YAAY,CAAA,GAAI;AAAC,GAC9E;AAEA,EAAA,OAAO,cAAA;AACT,CAAA;;;ACvFA,IAAO,oBAAA,GAAQ,iBAAA,CAAkB,SAAA,EAAW","file":"peam.adapter.js","sourcesContent":["import { readFileSync } from 'fs';\nimport type { NextAdapter } from 'next';\nimport { loggers } from 'peam/logger';\nimport {\n  loadRobotsTxt as baseLoadRobotsTxt,\n  createRobotsParser,\n  parseHTML,\n  shouldIncludePath,\n  type RobotsTxtResult,\n  type StructuredPage,\n} from 'peam/parser';\nimport { buildSearchIndex } from 'peam/search';\nimport { type ResolvedPeamAdapterConfig } from './config';\n\nconst log = loggers.adapter;\n\ninterface PrerenderOutput {\n  pathname: string;\n  fallback?: {\n    filePath: string;\n  };\n}\n\ninterface NextJS15Output extends PrerenderOutput {\n  type: string;\n}\n\ninterface NextJS16Outputs {\n  prerenders: Array<PrerenderOutput>;\n}\n\nfunction extractRobotsFromPrerender(prerender: PrerenderOutput): string | null {\n  try {\n    if (prerender.pathname !== '/robots.txt') {\n      return null;\n    }\n\n    if (prerender.fallback?.filePath) {\n      const content = readFileSync(prerender.fallback.filePath, 'utf-8');\n      return content;\n    }\n  } catch (error) {\n    log.error('Error extracting robots from prerender:', error);\n  }\n\n  return null;\n}\n\nfunction loadRobotsTxt(\n  projectDir: string,\n  prerenders: PrerenderOutput[],\n  robotsTxtPath?: string\n): RobotsTxtResult | null {\n  try {\n    let robotsContent: string | null = null;\n    let foundPath: string | null = null;\n\n    if (prerenders && prerenders.length > 0) {\n      for (const prerender of prerenders) {\n        const content = extractRobotsFromPrerender(prerender);\n        if (content) {\n          log.debug('Found dynamic robots.txt from prerenders');\n          robotsContent = content;\n          foundPath = prerender.pathname;\n          break;\n        }\n      }\n    }\n\n    if (!robotsContent) {\n      const searchPaths = ['public/robots.txt', 'app/robots.txt', 'src/app/robots.txt'];\n      const result = baseLoadRobotsTxt(projectDir, searchPaths, robotsTxtPath);\n      if (result) {\n        log.debug('Loaded robots.txt from:', result.path);\n        return result;\n      }\n      return null;\n    }\n\n    return {\n      parser: createRobotsParser(robotsContent),\n      path: foundPath || '',\n    };\n  } catch (error) {\n    log.error('Error loading robots.txt:', error);\n    return null;\n  }\n}\n\nexport function createPeamAdapter(config: ResolvedPeamAdapterConfig): NextAdapter {\n  return {\n    name: 'peam-adapter',\n\n    async onBuildComplete(ctx) {\n      log.debug('Extracting page content via adapter');\n\n      const outputs = ctx.outputs as NextJS15Output[] | NextJS16Outputs;\n      let prerenders: PrerenderOutput[];\n\n      if (Array.isArray(outputs)) {\n        prerenders = outputs.filter((output: NextJS15Output) => output.type === 'PRERENDER');\n      } else {\n        prerenders = outputs.prerenders || [];\n      }\n\n      log.debug('Total prerenders:', prerenders.length);\n\n      const projectDir = ctx.projectDir || process.cwd();\n\n      const robotsResult = config.respectRobotsTxt ? loadRobotsTxt(projectDir, prerenders, config.robotsTxtPath) : null;\n\n      if (robotsResult) {\n        log.debug('Using robots.txt from:', robotsResult.path);\n      }\n\n      const pages: Array<{\n        path: string;\n        htmlFile: string;\n        structuredPage: StructuredPage;\n        type: string;\n        runtime?: string;\n      }> = [];\n\n      for (const prerender of prerenders) {\n        const pathname = prerender.pathname;\n        let fallbackFilePath = prerender.fallback?.filePath;\n\n        if (!fallbackFilePath) {\n          continue;\n        }\n\n        // Fix for Next.js 15\n        if (fallbackFilePath?.endsWith('/.html')) {\n          fallbackFilePath = fallbackFilePath.replace('/.html', '/index.html');\n        }\n\n        const filterResult = shouldIncludePath(\n          pathname,\n          robotsResult?.parser ?? null,\n          config.exclude,\n          config.respectRobotsTxt\n        );\n\n        if (!filterResult.included) {\n          if (filterResult.reason === 'robots-txt') {\n            log.debug('Path excluded by robots.txt:', pathname);\n          } else if (filterResult.reason === 'exclude-pattern') {\n            log.debug('Path excluded by user pattern:', pathname);\n          }\n          continue;\n        }\n\n        try {\n          log.debug('Reading HTML from:', fallbackFilePath);\n\n          const html = readFileSync(fallbackFilePath, 'utf-8');\n          const structuredPage = parseHTML(html);\n\n          if (!structuredPage) {\n            log.warn('No content extracted from', pathname);\n            continue;\n          }\n\n          log.debug('Successfully extracted content from', pathname);\n          pages.push({\n            path: pathname,\n            htmlFile: fallbackFilePath.replace(projectDir + '/', ''),\n            structuredPage,\n            type: 'page',\n          });\n        } catch (error) {\n          log.error('Error processing', pathname, error);\n        }\n      }\n\n      log.debug('Creating search index...');\n      const searchIndexData = await buildSearchIndex(pages);\n\n      // Use the exporter to save the search index\n      await config.searchIndexExporter.export(searchIndexData);\n\n      log.debug('Saved search index via exporter');\n      log.debug('Extraction complete with total pages:', pages.length);\n    },\n  };\n}\n","import { createExporterFromConfig, type SearchExporterConfig, type SearchIndexExporter } from '@peam-ai/search';\nimport { NextConfig } from 'next';\n\nexport interface PeamConfig {\n  /**\n   * Search exporter configuration\n   * @default { type: 'fileBased', config: { indexPath: '.peam/index.json' } }\n   */\n  searchExporter?: SearchExporterConfig;\n  /**\n   * Whether to respect robots.txt rules when indexing pages\n   * @default true\n   */\n  respectRobotsTxt?: boolean;\n  /**\n   * Path to a custom robots.txt file relative to the project root\n   * If not specified, the adapter will look for static or dynamic robots.txt files in standard locations\n   * @example 'custom/robots.txt' or 'config/production-robots.txt'\n   * @default undefined\n   */\n  robotsTxtPath?: string;\n  /**\n   * Array of wildcard patterns to exclude from indexing\n   * Supports * (matches any characters except /), ** (matches any characters including /), and ? (single character)\n   * @example ['/admin/**', '/api/*', '/private-*']\n   * @default []\n   */\n  exclude?: string[];\n}\n\nexport type ResolvedPeamAdapterConfig = Required<Omit<PeamConfig, 'robotsTxtPath'>> & {\n  robotsTxtPath?: string;\n  searchIndexExporter: SearchIndexExporter;\n};\n\nconst defaultConfig = {\n  searchExporter: {\n    type: 'fileBased' as const,\n    config: { indexPath: '.peam/index.json' },\n  },\n  respectRobotsTxt: true,\n  robotsTxtPath: undefined,\n  exclude: [],\n} satisfies PeamConfig;\n\nexport function setNextConfig(nextConfig: NextConfig, peamConfig?: PeamConfig): void {\n  const envVars = {\n    PEAM_SEARCH_EXPORTER_TYPE: peamConfig?.searchExporter?.type ?? defaultConfig.searchExporter.type,\n    PEAM_SEARCH_EXPORTER_CONFIG:\n      JSON.stringify(peamConfig?.searchExporter?.config) ?? JSON.stringify(defaultConfig.searchExporter.config),\n    PEAM_RESPECT_ROBOTS_TXT: String(peamConfig?.respectRobotsTxt ?? defaultConfig.respectRobotsTxt),\n    PEAM_EXCLUDE: JSON.stringify(peamConfig?.exclude) ?? JSON.stringify(defaultConfig.exclude),\n    PEAM_ROBOTS_TXT_PATH: '',\n  };\n\n  if (peamConfig?.robotsTxtPath) {\n    envVars.PEAM_ROBOTS_TXT_PATH = String(peamConfig.robotsTxtPath);\n  }\n\n  // Set build time vars\n  Object.assign(process.env, envVars);\n\n  // Set runtime vars\n  nextConfig.env = {\n    ...nextConfig.env,\n    ...envVars,\n  };\n}\n\nexport const getConfig = (): ResolvedPeamAdapterConfig => {\n  if (!process.env.PEAM_SEARCH_EXPORTER_TYPE || !process.env.PEAM_SEARCH_EXPORTER_CONFIG) {\n    throw new Error(\n      'Peam configuration not found. Make sure withPeam() is properly configured in your next.config file.'\n    );\n  }\n\n  const searchExporterConfig: SearchExporterConfig = {\n    type: process.env.PEAM_SEARCH_EXPORTER_TYPE as SearchExporterConfig['type'],\n    config: JSON.parse(process.env.PEAM_SEARCH_EXPORTER_CONFIG),\n  };\n\n  const resolvedConfig = {\n    searchExporter: searchExporterConfig,\n    searchIndexExporter: createExporterFromConfig(searchExporterConfig),\n    respectRobotsTxt: process.env.PEAM_RESPECT_ROBOTS_TXT === 'true',\n    robotsTxtPath: process.env.PEAM_ROBOTS_TXT_PATH || undefined,\n    exclude: process.env.PEAM_EXCLUDE ? JSON.parse(process.env.PEAM_EXCLUDE) : [],\n  };\n\n  return resolvedConfig;\n};\n","import { createPeamAdapter } from './adapter';\nimport { getConfig } from './config';\n\nexport default createPeamAdapter(getConfig());\n"]}

package/dist/route.d.mts

@@ -1,11 +1,3 @@
-/**
- * Default POST handler using GPT-4o.
- *
- * @example
- * ```typescript
- * export { POST } from '@peam-ai/next/route';
- * ```
- */
 declare const POST: (req: Request) => Promise<Response>;
 
 export { POST };

package/dist/route.d.ts

@@ -1,11 +1,3 @@
-/**
- * Default POST handler using GPT-4o.
- *
- * @example
- * ```typescript
- * export { POST } from '@peam-ai/next/route';
- * ```
- */
 declare const POST: (req: Request) => Promise<Response>;
 
 export { POST };