extract-from-sitemap 0.0.4 → 0.0.6

package/cli.js

@@ -8,13 +8,20 @@ const http = require("http");
 const { URL, URLSearchParams } = require("url");
 const os = require("os");
 const { extractFromSitemap } = require("./mod.js");
+
+/**
+ * @typedef {Object} SourceConfig
+ * @property {string} [origin] - The origin URL to process (optional)
+ * @property {boolean} forceExtract - Whether to force extraction for this source
+ * @property {string} outDir - Output directory for this source's extracted files
+ * @property {Array<{title: string, description: string, url: string}>} [customUrls] - Custom URLs to extract for this source
+ */
+
 /**
  * @typedef {Object} Config
- * @property {string} outDir - Output directory for extracted files
- * @property {string[]} origins - Array of origin URLs to process
- * @property {Array<{title: string, description: string, url: string}>} customUrls - Custom URLs to extract
+ * @property {string} outDir - Top-level output directory for combined llms.txt
+ * @property {SourceConfig[]} sources - Array of source configurations
  * @property {boolean} keepOriginalUrls - Whether to keep original URL structure
- * @property {boolean} forceExtract - Whether to force extraction even if files exist
  */
 
 /**
@@ -209,11 +216,32 @@ async function loadConfig() {
     console.log(
       JSON.stringify(
         {
+          $schema: "https://extract.llmtext.com/llmtext.schema.json",
           outDir: "./docs",
-          origins: ["https://docs.example.com"],
-          customUrls: [],
+          sources: [
+            {
+              origin: "https://docs.parallel.ai",
+              forceExtract: false,
+              outDir: "./docs/parallel-docs",
+            },
+            {
+              origin: "https://parallel.ai",
+              forceExtract: true,
+              outDir: "./docs/parallel-main",
+            },
+            {
+              forceExtract: true,
+              outDir: "./docs/custom",
+              customUrls: [
+                {
+                  title: "Custom Page",
+                  description: "A custom page to extract",
+                  url: "https://example.com/page",
+                },
+              ],
+            },
+          ],
           keepOriginalUrls: false,
-          forceExtract: false,
         },
         null,
         2
@@ -227,13 +255,38 @@ async function loadConfig() {
 
     // Validate required fields
     if (!config.outDir) throw new Error("outDir is required");
-    if (!Array.isArray(config.origins))
-      throw new Error("origins must be an array");
+    if (!Array.isArray(config.sources))
+      throw new Error("sources must be an array");
+
+    // Validate source objects
+    for (const [index, sourceConfig] of config.sources.entries()) {
+      if (typeof sourceConfig !== "object" || sourceConfig === null) {
+        throw new Error(`sources[${index}] must be an object`);
+      }
+      if (!sourceConfig.outDir) {
+        throw new Error(`sources[${index}].outDir is required`);
+      }
+      if (typeof sourceConfig.forceExtract !== "boolean") {
+        throw new Error(`sources[${index}].forceExtract must be a boolean`);
+      }
+      // Either origin or customUrls must be provided
+      if (
+        !sourceConfig.origin &&
+        (!sourceConfig.customUrls || sourceConfig.customUrls.length === 0)
+      ) {
+        throw new Error(
+          `sources[${index}] must have either origin or customUrls`
+        );
+      }
+    }
 
     // Set defaults
-    config.customUrls = config.customUrls || [];
     config.keepOriginalUrls = config.keepOriginalUrls ?? false;
-    config.forceExtract = config.forceExtract ?? false;
+
+    // Set default customUrls for each source
+    for (const sourceConfig of config.sources) {
+      sourceConfig.customUrls = sourceConfig.customUrls || [];
+    }
 
     return config;
   } catch (error) {
@@ -370,10 +423,9 @@ function cleanupOldFiles(outDir, currentFiles, previousFiles) {
  * Process custom URLs through extraction API
  * @param {Array<{title: string, description: string, url: string}>} customUrls - Custom URLs to process
  * @param {string} apiKey - API key for authentication
- * @param {boolean} forceExtract - Whether to force extraction
  * @returns {Promise<Record<string, any>>} Extracted files
  */
-async function processCustomUrls(customUrls, apiKey, forceExtract) {
+async function processCustomUrls(customUrls, apiKey) {
   const files = {};
 
   for (const customUrl of customUrls) {
@@ -424,6 +476,47 @@ async function processCustomUrls(customUrls, apiKey, forceExtract) {
   return files;
 }
 
+/**
+ * Generate combined llms.txt from all sources
+ * @param {Array<{sourceName: string, files: Record<string, any>, origin?: string}>} allSources - All processed sources
+ * @returns {string} Combined llms.txt content
+ */
+function generateCombinedLlmsTxt(allSources) {
+  let combinedTxt =
+    "# Documentation Collection\n\n> Combined documentation from multiple sources\n\n";
+
+  for (const source of allSources) {
+    const sourceName = source.origin
+      ? new URL(
+          source.origin.startsWith("http")
+            ? source.origin
+            : `https://${source.origin}`
+        ).hostname
+      : source.sourceName;
+
+    combinedTxt += `## ${sourceName}\n\n`;
+
+    // Sort files by path for consistent ordering
+    const sortedFiles = Object.entries(source.files)
+      .filter(([path]) => path !== "/llms.txt")
+      .sort(([a], [b]) => a.localeCompare(b));
+
+    for (const [path, file] of sortedFiles) {
+      if (file.content || file.title) {
+        const title = file.title || path.replace(".md", "");
+        const description = file.description ? `: ${file.description}` : "";
+        combinedTxt += `- [${title}](${path.replace(".md", "")}) (${
+          file.tokens
+        } tokens)${description}\n`;
+      }
+    }
+
+    combinedTxt += "\n";
+  }
+
+  return combinedTxt;
+}
+
 /**
  * Clear stored API key credentials
  */
@@ -457,111 +550,143 @@ async function main() {
     const config = await loadConfig();
     const apiKey = await getApiKey();
 
-    // Ensure output directory exists
+    // Ensure top-level output directory exists
     fs.mkdirSync(config.outDir, { recursive: true });
 
-    // Load previous manifest
-    const previousManifest = loadManifest(config.outDir);
-    const currentFiles = [];
-
+    const allSources = [];
     let totalTokens = 0;
     let totalPages = 0;
     let totalErrors = 0;
 
-    // Process each origin
-    for (const origin of config.origins) {
-      console.log(`\n🌐 Processing origin: ${origin}`);
+    // Process each source
+    for (const [sourceIndex, sourceConfig] of config.sources.entries()) {
+      const sourceName = sourceConfig.origin
+        ? `source ${sourceIndex + 1} (${sourceConfig.origin})`
+        : `source ${sourceIndex + 1} (custom URLs)`;
+
+      console.log(
+        `\n🌐 Processing ${sourceName} (forceExtract: ${sourceConfig.forceExtract})`
+      );
+
+      // Ensure source output directory exists
+      fs.mkdirSync(sourceConfig.outDir, { recursive: true });
+
+      // Load previous manifest for this source
+      const previousManifest = loadManifest(sourceConfig.outDir);
+      const currentFiles = [];
+      let sourceFiles = {};
 
       try {
-        const result = await extractFromSitemap(
-          origin,
-          config.forceExtract,
-          apiKey
-        );
+        // Process origin if provided
+        if (sourceConfig.origin) {
+          const result = await extractFromSitemap(
+            sourceConfig.origin,
+            sourceConfig.forceExtract,
+            apiKey
+          );
+
+          console.log(
+            `✅ Extracted ${result.totalPages} pages with ${result.totalTokens} tokens`
+          );
+          if (result.errors > 0) {
+            console.log(`⚠️  ${result.errors} errors occurred`);
+          }
 
-        console.log(
-          `✅ Extracted ${result.totalPages} pages with ${result.totalTokens} tokens`
-        );
-        if (result.errors > 0) {
-          console.log(`⚠️  ${result.errors} errors occurred`);
+          sourceFiles = result.files;
+          totalTokens += result.totalTokens;
+          totalPages += result.totalPages;
+          totalErrors += result.errors;
         }
 
-        // Write files to disk
-        for (const [filePath, file] of Object.entries(result.files)) {
+        // Process custom URLs for this source
+        if (sourceConfig.customUrls && sourceConfig.customUrls.length > 0) {
+          console.log(
+            `📋 Processing ${sourceConfig.customUrls.length} custom URLs for this source...`
+          );
+          const customFiles = await processCustomUrls(
+            sourceConfig.customUrls,
+            apiKey
+          );
+
+          // Merge custom files with sitemap files
+          sourceFiles = { ...sourceFiles, ...customFiles };
+
+          for (const file of Object.values(customFiles)) {
+            totalTokens += file.tokens;
+            totalPages++;
+          }
+        }
+
+        // Write files to source directory
+        for (const [filePath, file] of Object.entries(sourceFiles)) {
           let filename = filePath;
 
-          if (!config.keepOriginalUrls) {
-            // Create domain-specific subdirectory
-            const domain = new URL(
-              origin.startsWith("http") ? origin : `https://${origin}`
-            ).hostname;
-            const domainDir = path.join(config.outDir, domain);
-            fs.mkdirSync(domainDir, { recursive: true });
-            filename = path.join(
-              domain,
-              filePath.startsWith("/") ? filePath.slice(1) : filePath
-            );
-          } else {
+          if (!config.keepOriginalUrls && sourceConfig.origin) {
+            // Use relative path within source directory
+            filename = filePath.startsWith("/") ? filePath.slice(1) : filePath;
+          } else if (!sourceConfig.origin) {
+            // For custom URL sources, use simple filename
             filename = filePath.startsWith("/") ? filePath.slice(1) : filePath;
           }
 
-          const fullFilePath = path.join(config.outDir, filename);
+          const fullFilePath = path.join(sourceConfig.outDir, filename);
           const fileDir = path.dirname(fullFilePath);
 
           fs.mkdirSync(fileDir, { recursive: true });
           fs.writeFileSync(fullFilePath, file.content);
           currentFiles.push(filename);
 
-          console.log(`📝 Wrote: ${filename} (${file.tokens} tokens)`);
+          console.log(
+            `📝 Wrote: ${path.join(sourceConfig.outDir, filename)} (${
+              file.tokens
+            } tokens)`
+          );
         }
 
-        totalTokens += result.totalTokens;
-        totalPages += result.totalPages;
-        totalErrors += result.errors;
+        // Clean up old files for this source
+        if (previousManifest.files.length > 0) {
+          cleanupOldFiles(
+            sourceConfig.outDir,
+            currentFiles,
+            previousManifest.files
+          );
+        }
+
+        // Save manifest for this source
+        const newManifest = {
+          files: currentFiles,
+          timestamp: new Date().toISOString(),
+        };
+        saveManifest(sourceConfig.outDir, newManifest);
+
+        // Add to all sources for combined llms.txt
+        allSources.push({
+          sourceName: `Source ${sourceIndex + 1}`,
+          origin: sourceConfig.origin,
+          files: sourceFiles,
+        });
       } catch (error) {
-        console.error(`❌ Error processing ${origin}:`, error.message);
+        console.error(`❌ Error processing ${sourceName}:`, error.message);
         totalErrors++;
       }
     }
 
-    // Process custom URLs
-    if (config.customUrls.length > 0) {
-      console.log(`\n📋 Processing ${config.customUrls.length} custom URLs...`);
-      const customFiles = await processCustomUrls(
-        config.customUrls,
-        apiKey,
-        config.forceExtract
-      );
-
-      for (const [filename, file] of Object.entries(customFiles)) {
-        const filePath = path.join(config.outDir, filename);
-        fs.writeFileSync(filePath, file.content);
-        currentFiles.push(filename);
-        totalTokens += file.tokens;
-        totalPages++;
-
-        console.log(`📝 Wrote: ${filename} (${file.tokens} tokens)`);
-      }
+    // Generate and write combined llms.txt to top-level outDir
+    if (allSources.length > 0) {
+      const combinedLlmsTxt = generateCombinedLlmsTxt(allSources);
+      const combinedLlmsTxtPath = path.join(config.outDir, "llms.txt");
+      fs.writeFileSync(combinedLlmsTxtPath, combinedLlmsTxt);
+      console.log(`\n📋 Generated combined llms.txt: ${combinedLlmsTxtPath}`);
     }
 
-    // Clean up old files
-    if (previousManifest.files.length > 0) {
-      cleanupOldFiles(config.outDir, currentFiles, previousManifest.files);
-    }
-
-    // Save new manifest
-    const newManifest = {
-      files: currentFiles,
-      timestamp: new Date().toISOString(),
-    };
-    saveManifest(config.outDir, newManifest);
-
     console.log("\n✨ Extraction completed!");
     console.log(`📊 Total: ${totalPages} pages, ${totalTokens} tokens`);
     if (totalErrors > 0) {
       console.log(`⚠️  Errors: ${totalErrors}`);
     }
-    console.log(`📁 Output directory: ${path.resolve(config.outDir)}`);
+    console.log(
+      `📁 Top-level output directory: ${path.resolve(config.outDir)}`
+    );
     console.log("\n💡 Use --clear-credentials to remove stored API key");
   } catch (error) {
     console.error("💥 Fatal error:", error.message);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "extract-from-sitemap",
   "bin": "cli.js",
-  "version": "0.0.4",
+  "version": "0.0.6",
   "main": "mod.js",
   "description": "A module and CLI that allows extracting all pages from a sitemap into markdown and a llms.txt, using Parallel.ai APIs.",
   "files": [