extract-from-sitemap 0.0.3 → 0.0.4

package/cli.js

@@ -7,7 +7,7 @@ const crypto = require("crypto");
 const http = require("http");
 const { URL, URLSearchParams } = require("url");
 const os = require("os");
-
+const { extractFromSitemap } = require("./mod.js");
 /**
  * @typedef {Object} Config
  * @property {string} outDir - Output directory for extracted files
@@ -410,6 +410,8 @@ async function processCustomUrls(customUrls, apiKey, forceExtract) {
             tokens: Math.round((extracted.full_content || "").length / 5),
           };
         }
+      } else {
+        throw new Error(`${response.status} - ${await response.statusText()}`);
       }
     } catch (error) {
       console.error(
@@ -438,27 +440,6 @@ async function clearCredentials() {
   }
 }
 
-/**
- * Extract content from sitemap (placeholder - you'll need to implement this)
- * @param {string} origin - The origin URL
- * @param {boolean} forceExtract - Whether to force extraction
- * @param {string} apiKey - API key for authentication
- * @returns {Promise<{totalPages: number, totalTokens: number, errors: number, files: Record<string, any>}>}
- */
-async function extractFromSitemap(origin, forceExtract, apiKey) {
-  // This is a placeholder - you'll need to implement the actual extraction logic
-  // or import it from your mod.js file
-  console.log(`Extracting from ${origin} (force: ${forceExtract})`);
-
-  // For now, return empty result
-  return {
-    totalPages: 0,
-    totalTokens: 0,
-    errors: 0,
-    files: {},
-  };
-}
-
 /**
  * Main function
  */

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "extract-from-sitemap",
   "bin": "cli.js",
-  "version": "0.0.3",
+  "version": "0.0.4",
   "main": "mod.js",
   "description": "A module and CLI that allows extracting all pages from a sitemap into markdown and a llms.txt, using Parallel.ai APIs.",
   "files": [