@peam-ai/next 0.1.4 → 0.1.5

package/dist/index.mjs

@@ -1,9 +1,7 @@
-import { readFileSync } from 'fs';
-import { loggers } from 'peam/logger';
-import { shouldIncludePath, parseHTML, loadRobotsTxt as loadRobotsTxt$1, createRobotsParser } from 'peam/parser';
-import { buildSearchIndex } from 'peam/search';
-import * as fs from 'fs/promises';
+import { createRequire } from 'module';
 import * as path from 'path';
+import * as fsSync from 'fs';
+import * as fs from 'fs/promises';
 
 var __create = Object.create;
 var __defProp = Object.defineProperty;
@@ -108,125 +106,6 @@ var require_picocolors = __commonJS({
     module2.exports.createColors = createColors;
   }
 });
-var log = loggers.adapter;
-function extractRobotsFromPrerender(prerender) {
-  var _a;
-  try {
-    if (prerender.pathname !== "/robots.txt") {
-      return null;
-    }
-    if ((_a = prerender.fallback) == null ? void 0 : _a.filePath) {
-      const content = readFileSync(prerender.fallback.filePath, "utf-8");
-      return content;
-    }
-  } catch (error) {
-    log.error("Error extracting robots from prerender:", error);
-  }
-  return null;
-}
-function loadRobotsTxt(projectDir, prerenders, robotsTxtPath) {
-  try {
-    let robotsContent = null;
-    let foundPath = null;
-    if (prerenders && prerenders.length > 0) {
-      for (const prerender of prerenders) {
-        const content = extractRobotsFromPrerender(prerender);
-        if (content) {
-          log.debug("Found dynamic robots.txt from prerenders");
-          robotsContent = content;
-          foundPath = prerender.pathname;
-          break;
-        }
-      }
-    }
-    if (!robotsContent) {
-      const searchPaths = ["public/robots.txt", "app/robots.txt", "src/app/robots.txt"];
-      const result = loadRobotsTxt$1(projectDir, searchPaths, robotsTxtPath);
-      if (result) {
-        log.debug("Loaded robots.txt from:", result.path);
-        return result;
-      }
-      return null;
-    }
-    return {
-      parser: createRobotsParser(robotsContent),
-      path: foundPath || ""
-    };
-  } catch (error) {
-    log.error("Error loading robots.txt:", error);
-    return null;
-  }
-}
-function createPeamAdapter(config) {
-  return {
-    name: "peam-adapter",
-    async onBuildComplete(ctx) {
-      var _a, _b;
-      log.debug("Extracting page content via adapter");
-      const outputs = ctx.outputs;
-      let prerenders;
-      if (Array.isArray(outputs)) {
-        prerenders = outputs.filter((output) => output.type === "PRERENDER");
-      } else {
-        prerenders = outputs.prerenders || [];
-      }
-      log.debug("Total prerenders:", prerenders.length);
-      const projectDir = ctx.projectDir || process.cwd();
-      const robotsResult = config.respectRobotsTxt ? loadRobotsTxt(projectDir, prerenders, config.robotsTxtPath) : null;
-      if (robotsResult) {
-        log.debug("Using robots.txt from:", robotsResult.path);
-      }
-      const pages = [];
-      for (const prerender of prerenders) {
-        const pathname = prerender.pathname;
-        let fallbackFilePath = (_a = prerender.fallback) == null ? void 0 : _a.filePath;
-        if (!fallbackFilePath) {
-          continue;
-        }
-        if (fallbackFilePath == null ? void 0 : fallbackFilePath.endsWith("/.html")) {
-          fallbackFilePath = fallbackFilePath.replace("/.html", "/index.html");
-        }
-        const filterResult = shouldIncludePath(
-          pathname,
-          (_b = robotsResult == null ? void 0 : robotsResult.parser) != null ? _b : null,
-          config.exclude,
-          config.respectRobotsTxt
-        );
-        if (!filterResult.included) {
-          if (filterResult.reason === "robots-txt") {
-            log.debug("Path excluded by robots.txt:", pathname);
-          } else if (filterResult.reason === "exclude-pattern") {
-            log.debug("Path excluded by user pattern:", pathname);
-          }
-          continue;
-        }
-        try {
-          log.debug("Reading HTML from:", fallbackFilePath);
-          const html = readFileSync(fallbackFilePath, "utf-8");
-          const structuredPage = parseHTML(html);
-          if (!structuredPage) {
-            log.warn("No content extracted from", pathname);
-            continue;
-          }
-          log.debug("Successfully extracted content from", pathname);
-          pages.push({
-            path: pathname,
-            htmlFile: fallbackFilePath.replace(projectDir + "/", ""),
-            structuredPage,
-            type: "page"
-          });
-        } catch (error) {
-          log.error("Error processing", pathname, error);
-        }
-      }
-      log.debug("Creating search index...");
-      const searchIndexData = await buildSearchIndex(pages);
-      await config.searchIndexExporter.export(searchIndexData);
-      log.debug("Saved search index via exporter");
-      log.debug("Extraction complete with total pages:", pages.length);
-    }
-  };
-}
 
 // ../logger/dist/index.mjs
 var import_picocolors = __toESM(require_picocolors());
@@ -270,7 +149,8 @@ function print(message, args, { namespace, color = "white" }) {
   const ns = namespace ? `${import_picocolors.default.dim(namespace)} ` : "";
   console.log(header + ns + colorize(String(message)), ...args);
 }
-var loggers2 = {
+var loggers = {
+  next: createLogger("peam:next"),
   search: createLogger("peam:search")
 };
 
@@ -2563,11 +2443,12 @@ var __async = (__this, __arguments, generator) => {
     step((generator = generator.apply(__this, __arguments)).next());
   });
 };
-var log3 = loggers2.search;
+var log3 = loggers.search;
 var FileBasedSearchIndexExporter = class {
   constructor(options) {
     this.cachedData = null;
-    this.baseDir = options.baseDir;
+    var _a;
+    this.baseDir = (_a = options.baseDir) != null ? _a : process.cwd();
     this.indexPath = options.indexPath;
   }
   getFullPath() {
@@ -2605,10 +2486,18 @@ var FileBasedSearchIndexExporter = class {
       return data;
     });
   }
-  export(data) {
-    return __async(this, null, function* () {
+  export(_0) {
+    return __async(this, arguments, function* (data, options = { override: true }) {
       const fullPath = this.getFullPath();
       try {
+        if (!(options == null ? void 0 : options.override)) {
+          try {
+            yield fs.access(fullPath);
+            log3.debug("Search index file already exists and override is false, skipping export:", fullPath);
+            return;
+          } catch (e) {
+          }
+        }
         const dir = path.dirname(fullPath);
         yield fs.mkdir(dir, { recursive: true });
         yield fs.writeFile(fullPath, JSON.stringify(data, null, 2), "utf-8");
@@ -2619,6 +2508,26 @@ var FileBasedSearchIndexExporter = class {
       }
     });
   }
+  exportSync(data, options = { override: true }) {
+    const fullPath = this.getFullPath();
+    try {
+      if (!(options == null ? void 0 : options.override)) {
+        try {
+          fsSync.accessSync(fullPath);
+          log3.debug("Search index file already exists and override is false, skipping export:", fullPath);
+          return;
+        } catch (e) {
+        }
+      }
+      const dir = path.dirname(fullPath);
+      fsSync.mkdirSync(dir, { recursive: true });
+      fsSync.writeFileSync(fullPath, JSON.stringify(data, null, 2), "utf-8");
+      log3.debug("Search index saved to file:", fullPath, "with", data.keys.length, "keys");
+    } catch (error) {
+      log3.error("Failed to save search index to file:", fullPath, error);
+      throw error;
+    }
+  }
 };
 function createExporterFromConfig(exporterConfig) {
   if (exporterConfig.type === "fileBased") {
@@ -2631,7 +2540,7 @@ function createExporterFromConfig(exporterConfig) {
 var defaultConfig = {
   searchExporter: {
     type: "fileBased",
-    config: { baseDir: process.cwd(), indexPath: ".peam/index.json" }
+    config: { indexPath: ".peam/index.json" }
   },
   respectRobotsTxt: true,
   exclude: []
@@ -2660,12 +2569,13 @@ var getConfig = () => {
       "Peam configuration not found. Make sure withPeam() is properly configured in your next.config file."
     );
   }
-  const searchExporter = {
+  const searchExporterConfig = {
     type: process.env.PEAM_SEARCH_EXPORTER_TYPE,
     config: JSON.parse(process.env.PEAM_SEARCH_EXPORTER_CONFIG)
   };
   const resolvedConfig = {
-    searchIndexExporter: createExporterFromConfig(searchExporter),
+    searchExporter: searchExporterConfig,
+    searchIndexExporter: createExporterFromConfig(searchExporterConfig),
     respectRobotsTxt: process.env.PEAM_RESPECT_ROBOTS_TXT === "true",
     robotsTxtPath: process.env.PEAM_ROBOTS_TXT_PATH || void 0,
     exclude: process.env.PEAM_EXCLUDE ? JSON.parse(process.env.PEAM_EXCLUDE) : []
@@ -2674,19 +2584,117 @@ var getConfig = () => {
 };
 
 // src/withPeam.ts
+var require2 = createRequire(process.cwd() + "/");
+var log4 = loggers.next;
+var isProd = process.env.NODE_ENV === "production";
+function getNextVersion() {
+  try {
+    const [major, minor] = require2("next/package.json").version.split(".", 2).map(Number);
+    return { major, minor };
+  } catch (error) {
+    log4.error("Could not resolve Next.js version.", error);
+    return void 0;
+  }
+}
+function addStubIndex() {
+  var _a, _b, _c;
+  try {
+    if (!isProd) {
+      return;
+    }
+    const config = getConfig();
+    if (((_a = config.searchExporter) == null ? void 0 : _a.type) !== "fileBased") {
+      return;
+    }
+    const stubData = { keys: [], data: {} };
+    (_c = (_b = config.searchIndexExporter).exportSync) == null ? void 0 : _c.call(_b, stubData, { override: false });
+  } catch (error) {
+    log4.error("Failed to create stub index:", error);
+  }
+}
+function addAdapter(config) {
+  const nextVersion = getNextVersion();
+  if (!nextVersion || nextVersion.major < 16) {
+    log4.warn(
+      "Peam adapter requires Next.js 16 or higher, skipping adapter configuration. Make sure the postbuild script is set up correctly, See more here: https://peam.ai/docs."
+    );
+    return config;
+  }
+  return {
+    ...config,
+    experimental: {
+      ...config.experimental,
+      adapterPath: require2.resolve(path.join(__dirname, "peam.adapter.js"))
+    }
+  };
+}
+function addOutputFileTracing(nextConfig, peamConfig) {
+  var _a, _b, _c, _d;
+  nextConfig = { ...nextConfig };
+  if (((_a = peamConfig.searchExporter) == null ? void 0 : _a.type) !== "fileBased") {
+    return nextConfig;
+  }
+  const exporterConfig = peamConfig.searchExporter.config;
+  const indexDir = path.dirname(exporterConfig.indexPath);
+  const tracingConfig = {
+    "/api/peam": [`./${indexDir}/**/*`]
+  };
+  const nextVersion = getNextVersion();
+  if (!nextVersion) {
+    log4.warn(
+      "Could not determine Next.js version. Adding outputFileTracingIncludes to both experimental and root config."
+    );
+    const existingExperimentalTracing = nextConfig.experimental && typeof nextConfig.experimental === "object" && "outputFileTracingIncludes" in nextConfig.experimental ? nextConfig.experimental.outputFileTracingIncludes : void 0;
+    if (nextConfig.experimental) {
+      Object.assign(nextConfig.experimental, {
+        outputFileTracingIncludes: {
+          ...existingExperimentalTracing || {},
+          ...tracingConfig
+        }
+      });
+    }
+    const existingRootTracing = (_b = nextConfig.outputFileTracingIncludes) != null ? _b : void 0;
+    Object.assign(nextConfig, {
+      outputFileTracingIncludes: {
+        ...existingRootTracing || {},
+        ...tracingConfig
+      }
+    });
+  } else if (nextVersion.major < 15) {
+    const existingTracing = typeof nextConfig.experimental === "object" && "outputFileTracingIncludes" in nextConfig.experimental ? (_c = nextConfig.experimental) == null ? void 0 : _c.outputFileTracingIncludes : void 0;
+    if (nextConfig.experimental) {
+      Object.assign(nextConfig.experimental, {
+        outputFileTracingIncludes: {
+          ...existingTracing || {},
+          ...tracingConfig
+        }
+      });
+    }
+  } else {
+    const existingTracing = (_d = nextConfig.outputFileTracingIncludes) != null ? _d : void 0;
+    Object.assign(nextConfig, {
+      outputFileTracingIncludes: {
+        ...existingTracing || {},
+        ...tracingConfig
+      }
+    });
+  }
+  return nextConfig;
+}
 function withPeam(peamConfig) {
   return function(nextConfig = {}) {
     setNextConfig(nextConfig, peamConfig);
-    return {
-      ...nextConfig,
-      experimental: {
-        ...nextConfig.experimental,
-        adapterPath: __require.resolve("../dist/peam.adapter.js")
-      }
-    };
+    addStubIndex();
+    let updatedNextConfig = { ...nextConfig };
+    updatedNextConfig = addAdapter(updatedNextConfig);
+    updatedNextConfig = addOutputFileTracing(updatedNextConfig, getConfig());
+    return updatedNextConfig;
   };
 }
 
-export { createPeamAdapter, getConfig, setNextConfig, withPeam };
+// src/index.ts
+var index_default = withPeam;
+
+export { index_default as default };
 //# sourceMappingURL=index.mjs.map
 //# sourceMappingURL=index.mjs.map