extract-from-sitemap 0.0.7 → 0.0.9

package/cli.js

@@ -278,6 +278,9 @@ async function loadConfig() {
     if (!Array.isArray(config.sources))
       throw new Error("sources must be an array");
 
+    // Resolve top-level outDir to absolute path
+    const topLevelOutDir = path.resolve(config.outDir);
+
     // Validate source objects
     for (const [index, sourceConfig] of config.sources.entries()) {
       if (typeof sourceConfig !== "object" || sourceConfig === null) {
@@ -292,6 +295,22 @@ async function loadConfig() {
       sourceConfig.keepOriginalUrls = sourceConfig.keepOriginalUrls ?? false;
       sourceConfig.customUrls = sourceConfig.customUrls || [];
 
+      // Default outDir to top-level outDir if not specified
+      if (!sourceConfig.outDir) {
+        sourceConfig.outDir = config.outDir;
+      }
+
+      // Validate outDir is within top-level outDir (unless keepOriginalUrls is true)
+      if (!sourceConfig.keepOriginalUrls) {
+        const resolvedSourceOutDir = path.resolve(sourceConfig.outDir);
+
+        if (!resolvedSourceOutDir.startsWith(topLevelOutDir)) {
+          throw new Error(
+            `sources[${index}].outDir (${sourceConfig.outDir}) must be within the top-level outDir (${config.outDir})`
+          );
+        }
+      }
+
       // Either origin or customUrls must be provided
       if (
         !sourceConfig.origin &&
@@ -302,13 +321,6 @@ async function loadConfig() {
         );
       }
 
-      // outDir is required unless keepOriginalUrls is true
-      if (!sourceConfig.outDir && !sourceConfig.keepOriginalUrls) {
-        throw new Error(
-          `sources[${index}].outDir is required when keepOriginalUrls is false`
-        );
-      }
-
       // Validate customUrls
       for (const [urlIndex, customUrl] of (
         sourceConfig.customUrls || []
@@ -514,9 +526,27 @@ async function processCustomUrls(customUrls, apiKey) {
   return files;
 }
 
+/**
+ * Get path prefix for links in llms.txt
+ * @param {string} topLevelOutDir - Top-level output directory
+ * @param {string} sourceOutDir - Source-specific output directory
+ * @returns {string} Path prefix for links
+ */
+function getPathPrefix(topLevelOutDir, sourceOutDir) {
+  const resolvedTopLevel = path.resolve(topLevelOutDir);
+  const resolvedSource = path.resolve(sourceOutDir);
+
+  if (resolvedSource === resolvedTopLevel) {
+    return "";
+  }
+
+  const relativePath = path.relative(resolvedTopLevel, resolvedSource);
+  return relativePath ? relativePath + "/" : "";
+}
+
 /**
  * Generate combined llms.txt from all sources
- * @param {Array<{title: string, files: Record<string, any>, keepOriginalUrls?: boolean}>} allSources - All processed sources
+ * @param {Array<{title: string, files: Record<string, any>, keepOriginalUrls?: boolean, pathPrefix: string}>} allSources - All processed sources
  * @returns {string} Combined llms.txt content
  */
 function generateCombinedLlmsTxt(allSources) {
@@ -536,10 +566,13 @@ function generateCombinedLlmsTxt(allSources) {
         const title = file.title || path.replace(".md", "");
         const description = file.description ? `: ${file.description}` : "";
 
-        // If keepOriginalUrls is true, link to the original URL, otherwise link to the local file
-        const link = source.keepOriginalUrls
-          ? file.originalUrl
-          : path.replace(".md", "");
+        // Generate link based on keepOriginalUrls and pathPrefix
+        let link;
+        if (source.keepOriginalUrls) {
+          link = file.originalUrl;
+        } else {
+          link = source.pathPrefix + path;
+        }
 
         combinedTxt += `- [${title}](${link}) (${file.tokens} tokens)${description}\n`;
       }
@@ -600,13 +633,13 @@ async function main() {
         `\n🌐 Processing ${sourceName} (forceExtract: ${sourceConfig.forceExtract}, keepOriginalUrls: ${sourceConfig.keepOriginalUrls})`
       );
 
-      // Only ensure source output directory exists if not keeping original URLs
-      if (!sourceConfig.keepOriginalUrls && sourceConfig.outDir) {
+      // Ensure source output directory exists (if not keeping original URLs)
+      if (!sourceConfig.keepOriginalUrls) {
         fs.mkdirSync(sourceConfig.outDir, { recursive: true });
       }
 
-      // Load previous manifest for this source (only if we have an outDir)
-      const previousManifest = sourceConfig.outDir
+      // Load previous manifest for this source (only if we have an outDir and not keeping original URLs)
+      const previousManifest = !sourceConfig.keepOriginalUrls
         ? loadManifest(sourceConfig.outDir)
         : { files: [], timestamp: new Date().toISOString() };
       const currentFiles = [];
@@ -654,7 +687,7 @@ async function main() {
         }
 
         // Write files to source directory (only if not keeping original URLs)
-        if (!sourceConfig.keepOriginalUrls && sourceConfig.outDir) {
+        if (!sourceConfig.keepOriginalUrls) {
           for (const [filePath, file] of Object.entries(sourceFiles)) {
             let filename = filePath.startsWith("/")
               ? filePath.slice(1)
@@ -695,11 +728,17 @@ async function main() {
           );
         }
 
+        // Calculate path prefix for this source
+        const pathPrefix = sourceConfig.keepOriginalUrls
+          ? ""
+          : getPathPrefix(config.outDir, sourceConfig.outDir);
+
         // Add to all sources for combined llms.txt
         allSources.push({
           title: sourceConfig.title,
           files: sourceFiles,
           keepOriginalUrls: sourceConfig.keepOriginalUrls,
+          pathPrefix: pathPrefix,
         });
       } catch (error) {
         console.error(`❌ Error processing ${sourceName}:`, error.message);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "extract-from-sitemap",
   "bin": "cli.js",
-  "version": "0.0.7",
+  "version": "0.0.9",
   "main": "mod.js",
   "description": "A module and CLI that allows extracting all pages from a sitemap into markdown and a llms.txt, using Parallel.ai APIs.",
   "files": [