extract-from-sitemap 0.0.2 → 0.0.4

package/{cli.ts → cli.js}

@@ -1,52 +1,47 @@
-#!/usr/bin/env bun
-/// <reference types="@types/bun" />
-/// <reference lib="esnext" />
-
-import {
-  existsSync,
-  readFileSync,
-  writeFileSync,
-  mkdirSync,
-  rmSync,
-  readdirSync,
-} from "fs";
-import { join, dirname, resolve } from "path";
-import { extractFromSitemap } from "./mod.js";
-import { secrets } from "bun";
-
-interface Config {
-  outDir: string;
-  origins: string[];
-  customUrls: Array<{
-    title: string;
-    description: string;
-    url: string;
-  }>;
-  keepOriginalUrls: boolean;
-  forceExtract: boolean;
-}
-
-interface Manifest {
-  files: string[];
-  timestamp: string;
-}
-
-const SECRETS_SERVICE = "extract-from-sitemap-cli";
-const SECRETS_KEY = "parallel-api-key";
-
+#!/usr/bin/env node
+
+const fs = require("fs");
+const path = require("path");
+const { spawn } = require("child_process");
+const crypto = require("crypto");
+const http = require("http");
+const { URL, URLSearchParams } = require("url");
+const os = require("os");
+const { extractFromSitemap } = require("./mod.js");
+/**
+ * @typedef {Object} Config
+ * @property {string} outDir - Output directory for extracted files
+ * @property {string[]} origins - Array of origin URLs to process
+ * @property {Array<{title: string, description: string, url: string}>} customUrls - Custom URLs to extract
+ * @property {boolean} keepOriginalUrls - Whether to keep original URL structure
+ * @property {boolean} forceExtract - Whether to force extraction even if files exist
+ */
+
+/**
+ * @typedef {Object} Manifest
+ * @property {string[]} files - List of generated files
+ * @property {string} timestamp - Timestamp of last generation
+ */
+
+const CREDENTIALS_DIR = path.join(os.homedir(), ".llmtext");
+const API_KEY_FILE = path.join(CREDENTIALS_DIR, "api-key");
+
+/**
+ * OAuth handler for Parallel.ai API key authentication
+ */
 class OAuth {
-  private clientId: string;
-  private redirectUri: string;
-  private scope: string;
-  private server?: Bun.Server;
-
   constructor() {
     this.clientId = "extract-from-sitemap-cli";
     this.redirectUri = "http://localhost:3737/callback";
     this.scope = "key:read";
+    this.server = null;
   }
 
-  async getApiKey(): Promise<string> {
+  /**
+   * Get API key through OAuth flow
+   * @returns {Promise<string>} The API key
+   */
+  async getApiKey() {
     console.log("🔐 Starting OAuth flow...");
 
     // Generate PKCE parameters
@@ -62,7 +57,7 @@ class OAuth {
     authUrl.searchParams.set("code_challenge_method", "S256");
     authUrl.searchParams.set("state", Math.random().toString(36));
 
-    console.log(`\n📖 Opening browser for authorization...`);
+    console.log("\n📖 Opening browser for authorization...");
 
     // Open browser automatically
     await this.openBrowser(authUrl.toString());
@@ -97,13 +92,14 @@ class OAuth {
     return access_token;
   }
 
-  private async openBrowser(url: string): Promise<void> {
+  /**
+   * Open browser to authorization URL
+   * @param {string} url - The authorization URL
+   */
+  async openBrowser(url) {
     try {
-      const { spawn } = require("child_process");
       const platform = process.platform;
-
-      let command: string;
-      let args: string[];
+      let command, args;
 
       if (platform === "darwin") {
         command = "open";
@@ -119,90 +115,93 @@ class OAuth {
 
       spawn(command, args, { detached: true, stdio: "ignore" });
     } catch (error) {
-      console.log(`\n📖 Please visit this URL to authorize the application:`);
+      console.log("\n📖 Please visit this URL to authorize the application:");
       console.log(`${url}\n`);
     }
   }
 
-  private async generatePKCE(): Promise<{
-    codeVerifier: string;
-    codeChallenge: string;
-  }> {
-    const codeVerifier = btoa(
-      String.fromCharCode(...crypto.getRandomValues(new Uint8Array(32)))
-    ).replace(/[+/=]/g, (m) => ({ "+": "-", "/": "_", "=": "" }[m]));
-
-    const hash = await crypto.subtle.digest(
-      "SHA-256",
-      new TextEncoder().encode(codeVerifier)
-    );
-    const codeChallenge = btoa(
-      String.fromCharCode(...new Uint8Array(hash))
-    ).replace(/[+/=]/g, (m) => ({ "+": "-", "/": "_", "=": "" }[m]));
-
-    return { codeVerifier, codeChallenge };
+  /**
+   * Generate PKCE code verifier and challenge
+   * @returns {Promise<{codeVerifier: string, codeChallenge: string}>}
+   */
+  async generatePKCE() {
+    const codeVerifier = crypto.randomBytes(32).toString("base64url");
+    const hash = crypto
+      .createHash("sha256")
+      .update(codeVerifier)
+      .digest("base64url");
+
+    return {
+      codeVerifier,
+      codeChallenge: hash,
+    };
   }
 
-  private async startCallbackServer(): Promise<string> {
+  /**
+   * Start HTTP server to catch OAuth callback
+   * @returns {Promise<string>} The authorization code
+   */
+  async startCallbackServer() {
     return new Promise((resolve, reject) => {
-      this.server = Bun.serve({
-        port: 3737,
-        fetch: (req) => {
-          const url = new URL(req.url);
-
-          if (url.pathname === "/callback") {
-            const code = url.searchParams.get("code");
-            const error = url.searchParams.get("error");
-
-            if (error) {
-              reject(new Error(`OAuth error: ${error}`));
-              return new Response(
-                "Error occurred. You can close this window.",
-                { status: 400 }
-              );
-            }
-
-            if (code) {
-              resolve(code);
-              // Don't stop server here - let the cleanup happen in the finally block
-              return new Response(
-                "✅ Authorization successful! You can close this window and return to the terminal.",
-                {
-                  headers: {
-                    "Content-Type": "text/html",
-                  },
-                }
-              );
-            }
+      this.server = http.createServer((req, res) => {
+        const url = new URL(req.url, `http://${req.headers.host}`);
+
+        if (url.pathname === "/callback") {
+          const code = url.searchParams.get("code");
+          const error = url.searchParams.get("error");
+
+          if (error) {
+            reject(new Error(`OAuth error: ${error}`));
+            res.writeHead(400);
+            res.end("Error occurred. You can close this window.");
+            return;
           }
 
-          return new Response("Invalid request", { status: 404 });
-        },
+          if (code) {
+            resolve(code);
+            res.writeHead(200, { "Content-Type": "text/html" });
+            res.end(
+              "✅ Authorization successful! You can close this window and return to the terminal."
+            );
+            return;
+          }
+        }
+
+        res.writeHead(404);
+        res.end("Invalid request");
       });
 
+      this.server.listen(3737);
+
       // Timeout after 5 minutes
       setTimeout(() => {
         this.stopServer();
         reject(new Error("OAuth flow timed out"));
       }, 300000);
     }).finally(() => {
-      // Ensure server is stopped after promise resolves or rejects
       this.stopServer();
     });
   }
 
-  private stopServer(): void {
+  /**
+   * Stop the callback server
+   */
+  stopServer() {
     if (this.server) {
-      this.server.stop();
-      this.server = undefined;
+      this.server.close();
+      this.server = null;
     }
   }
 }
 
-async function loadConfig(): Promise<Config> {
-  const configPath = resolve("llmtext.json");
+/**
+ * Load configuration from llmtext.json
+ * @returns {Promise<Config>} The configuration object
+ */
+async function loadConfig() {
+  const configPath = path.resolve("llmtext.json");
 
-  if (!existsSync(configPath)) {
+  if (!fs.existsSync(configPath)) {
     console.error(
       "❌ llmtext.json not found. Please create a configuration file."
     );
@@ -224,7 +223,7 @@ async function loadConfig(): Promise<Config> {
   }
 
   try {
-    const config = JSON.parse(readFileSync(configPath, "utf8")) as Config;
+    const config = JSON.parse(fs.readFileSync(configPath, "utf8"));
 
     // Validate required fields
     if (!config.outDir) throw new Error("outDir is required");
@@ -243,28 +242,56 @@ async function loadConfig(): Promise<Config> {
   }
 }
 
-async function getApiKey(): Promise<string> {
-  // Check if we have a stored API key in the keychain
+/**
+ * Store API key in ~/.llmtext/api-key
+ * @param {string} apiKey - The API key to store
+ */
+function storeApiKey(apiKey) {
   try {
-    const storedKey = await secrets.get({
-      service: SECRETS_SERVICE,
-      name: SECRETS_KEY,
-    });
+    fs.mkdirSync(CREDENTIALS_DIR, { recursive: true });
+    fs.writeFileSync(API_KEY_FILE, apiKey, { mode: 0o600 }); // Only owner can read
+    console.log("💾 API key stored securely in ~/.llmtext/api-key");
+  } catch (error) {
+    console.warn("⚠️  Could not store API key:", error.message);
+  }
+}
 
-    if (storedKey) {
-      console.log("🔑 Using stored API key from keychain");
-      return storedKey;
+/**
+ * Load API key from ~/.llmtext/api-key
+ * @returns {string|null} The stored API key or null if not found
+ */
+function loadStoredApiKey() {
+  try {
+    if (fs.existsSync(API_KEY_FILE)) {
+      const apiKey = fs.readFileSync(API_KEY_FILE, "utf8").trim();
+      if (apiKey) {
+        console.log("🔑 Using stored API key from ~/.llmtext/api-key");
+        return apiKey;
+      }
     }
   } catch (error) {
-    console.warn("⚠️  Could not access keychain:", error.message);
+    console.warn("⚠️  Could not read stored API key:", error.message);
+  }
+  return null;
+}
+
+/**
+ * Get API key from various sources or start OAuth flow
+ * @returns {Promise<string>} The API key
+ */
+async function getApiKey() {
+  // Check stored API key first
+  const storedKey = loadStoredApiKey();
+  if (storedKey) {
+    return storedKey;
   }
 
-  // Check environment variables as fallback
+  // Check environment variables
   let apiKey = process.env.PARALLEL_API_KEY;
 
-  if (!apiKey && existsSync(".env")) {
+  if (!apiKey && fs.existsSync(".env")) {
     // Try to load from .env file
-    const envContent = readFileSync(".env", "utf8");
+    const envContent = fs.readFileSync(".env", "utf8");
     const match = envContent.match(/^PARALLEL_API_KEY=(.+)$/m);
     if (match) {
       apiKey = match[1].trim();
@@ -273,17 +300,7 @@ async function getApiKey(): Promise<string> {
 
   if (apiKey) {
     console.log("🔑 Using API key from environment");
-    // Store it in keychain for future use
-    try {
-      await secrets.set({
-        service: SECRETS_SERVICE,
-        name: SECRETS_KEY,
-        value: apiKey,
-      });
-      console.log("💾 API key stored in keychain for future use");
-    } catch (error) {
-      console.warn("⚠️  Could not store API key in keychain:", error.message);
-    }
+    storeApiKey(apiKey);
     return apiKey;
   }
 
@@ -292,57 +309,55 @@ async function getApiKey(): Promise<string> {
   const oauth = new OAuth();
   const newApiKey = await oauth.getApiKey();
 
-  // Store the new API key in keychain
-  try {
-    await secrets.set({
-      service: SECRETS_SERVICE,
-      name: SECRETS_KEY,
-      value: newApiKey,
-    });
-    console.log("💾 API key stored securely in keychain");
-  } catch (error) {
-    console.warn("⚠️  Could not store API key in keychain:", error.message);
-    console.log(
-      "💡 You may need to set PARALLEL_API_KEY environment variable for future runs"
-    );
-  }
-
+  storeApiKey(newApiKey);
   return newApiKey;
 }
 
-function loadManifest(outDir: string): Manifest {
-  const manifestPath = join(outDir, "llmtext-manifest.json");
+/**
+ * Load manifest file
+ * @param {string} outDir - Output directory
+ * @returns {Manifest} The manifest object
+ */
+function loadManifest(outDir) {
+  const manifestPath = path.join(outDir, "llmtext-manifest.json");
 
-  if (!existsSync(manifestPath)) {
+  if (!fs.existsSync(manifestPath)) {
     return { files: [], timestamp: new Date().toISOString() };
   }
 
   try {
-    return JSON.parse(readFileSync(manifestPath, "utf8"));
+    return JSON.parse(fs.readFileSync(manifestPath, "utf8"));
   } catch {
     return { files: [], timestamp: new Date().toISOString() };
   }
 }
 
-function saveManifest(outDir: string, manifest: Manifest): void {
-  const manifestPath = join(outDir, "llmtext-manifest.json");
-  writeFileSync(manifestPath, JSON.stringify(manifest, null, 2));
+/**
+ * Save manifest file
+ * @param {string} outDir - Output directory
+ * @param {Manifest} manifest - The manifest to save
+ */
+function saveManifest(outDir, manifest) {
+  const manifestPath = path.join(outDir, "llmtext-manifest.json");
+  fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2));
 }
 
-function cleanupOldFiles(
-  outDir: string,
-  currentFiles: string[],
-  previousFiles: string[]
-): void {
+/**
+ * Clean up old files that are no longer generated
+ * @param {string} outDir - Output directory
+ * @param {string[]} currentFiles - Currently generated files
+ * @param {string[]} previousFiles - Previously generated files
+ */
+function cleanupOldFiles(outDir, currentFiles, previousFiles) {
   const filesToRemove = previousFiles.filter(
     (file) => !currentFiles.includes(file)
   );
 
   for (const file of filesToRemove) {
-    const filePath = join(outDir, file);
+    const filePath = path.join(outDir, file);
     try {
-      if (existsSync(filePath)) {
-        rmSync(filePath);
+      if (fs.existsSync(filePath)) {
+        fs.rmSync(filePath);
         console.log(`🗑️  Removed old file: ${file}`);
       }
     } catch (error) {
@@ -351,18 +366,20 @@ function cleanupOldFiles(
   }
 }
 
-async function processCustomUrls(
-  customUrls: Array<{ title: string; description: string; url: string }>,
-  apiKey: string,
-  forceExtract: boolean
-): Promise<Record<string, any>> {
-  const files: Record<string, any> = {};
+/**
+ * Process custom URLs through extraction API
+ * @param {Array<{title: string, description: string, url: string}>} customUrls - Custom URLs to process
+ * @param {string} apiKey - API key for authentication
+ * @param {boolean} forceExtract - Whether to force extraction
+ * @returns {Promise<Record<string, any>>} Extracted files
+ */
+async function processCustomUrls(customUrls, apiKey, forceExtract) {
+  const files = {};
 
   for (const customUrl of customUrls) {
     console.log(`📄 Processing custom URL: ${customUrl.url}`);
 
     try {
-      // For custom URLs, we need to extract them individually
       const response = await fetch("https://api.parallel.ai/v1beta/extract", {
         method: "POST",
         headers: {
@@ -393,6 +410,8 @@ async function processCustomUrls(
             tokens: Math.round((extracted.full_content || "").length / 5),
           };
         }
+      } else {
+        throw new Error(`${response.status} - ${await response.statusText()}`);
       }
     } catch (error) {
       console.error(
@@ -405,16 +424,14 @@ async function processCustomUrls(
   return files;
 }
 
-// Add command for clearing stored credentials
-async function clearCredentials(): Promise<void> {
+/**
+ * Clear stored API key credentials
+ */
+async function clearCredentials() {
   try {
-    const deleted = await secrets.delete({
-      service: SECRETS_SERVICE,
-      name: SECRETS_KEY,
-    });
-
-    if (deleted) {
-      console.log("✅ Cleared stored API key from keychain");
+    if (fs.existsSync(API_KEY_FILE)) {
+      fs.unlinkSync(API_KEY_FILE);
+      console.log("✅ Cleared stored API key from ~/.llmtext/api-key");
     } else {
       console.log("ℹ️  No stored API key found to clear");
     }
@@ -423,6 +440,9 @@ async function clearCredentials(): Promise<void> {
   }
 }
 
+/**
+ * Main function
+ */
 async function main() {
   console.log("🚀 Extract from Sitemap CLI");
 
@@ -438,11 +458,11 @@ async function main() {
     const apiKey = await getApiKey();
 
     // Ensure output directory exists
-    mkdirSync(config.outDir, { recursive: true });
+    fs.mkdirSync(config.outDir, { recursive: true });
 
     // Load previous manifest
     const previousManifest = loadManifest(config.outDir);
-    const currentFiles: string[] = [];
+    const currentFiles = [];
 
     let totalTokens = 0;
     let totalPages = 0;
@@ -467,29 +487,29 @@ async function main() {
         }
 
         // Write files to disk
-        for (const [path, file] of Object.entries(result.files)) {
-          let filename = path;
+        for (const [filePath, file] of Object.entries(result.files)) {
+          let filename = filePath;
 
           if (!config.keepOriginalUrls) {
             // Create domain-specific subdirectory
             const domain = new URL(
               origin.startsWith("http") ? origin : `https://${origin}`
             ).hostname;
-            const domainDir = join(config.outDir, domain);
-            mkdirSync(domainDir, { recursive: true });
-            filename = join(
+            const domainDir = path.join(config.outDir, domain);
+            fs.mkdirSync(domainDir, { recursive: true });
+            filename = path.join(
               domain,
-              path.startsWith("/") ? path.slice(1) : path
+              filePath.startsWith("/") ? filePath.slice(1) : filePath
             );
           } else {
-            filename = path.startsWith("/") ? path.slice(1) : path;
+            filename = filePath.startsWith("/") ? filePath.slice(1) : filePath;
           }
 
-          const filePath = join(config.outDir, filename);
-          const fileDir = dirname(filePath);
+          const fullFilePath = path.join(config.outDir, filename);
+          const fileDir = path.dirname(fullFilePath);
 
-          mkdirSync(fileDir, { recursive: true });
-          writeFileSync(filePath, file.content);
+          fs.mkdirSync(fileDir, { recursive: true });
+          fs.writeFileSync(fullFilePath, file.content);
           currentFiles.push(filename);
 
           console.log(`📝 Wrote: ${filename} (${file.tokens} tokens)`);
@@ -514,8 +534,8 @@ async function main() {
       );
 
       for (const [filename, file] of Object.entries(customFiles)) {
-        const filePath = join(config.outDir, filename);
-        writeFileSync(filePath, file.content);
+        const filePath = path.join(config.outDir, filename);
+        fs.writeFileSync(filePath, file.content);
         currentFiles.push(filename);
         totalTokens += file.tokens;
         totalPages++;
@@ -530,25 +550,34 @@ async function main() {
     }
 
     // Save new manifest
-    const newManifest: Manifest = {
+    const newManifest = {
       files: currentFiles,
       timestamp: new Date().toISOString(),
     };
     saveManifest(config.outDir, newManifest);
 
-    console.log(`\n✨ Extraction completed!`);
+    console.log("\n✨ Extraction completed!");
     console.log(`📊 Total: ${totalPages} pages, ${totalTokens} tokens`);
     if (totalErrors > 0) {
       console.log(`⚠️  Errors: ${totalErrors}`);
     }
-    console.log(`📁 Output directory: ${resolve(config.outDir)}`);
-    console.log(`\n💡 Use --clear-credentials to remove stored API key`);
+    console.log(`📁 Output directory: ${path.resolve(config.outDir)}`);
+    console.log("\n💡 Use --clear-credentials to remove stored API key");
   } catch (error) {
     console.error("💥 Fatal error:", error.message);
     process.exit(1);
   }
 }
 
-if (import.meta.main) {
+// Run main function if this file is executed directly
+if (require.main === module) {
   main();
 }
+
+module.exports = {
+  OAuth,
+  loadConfig,
+  getApiKey,
+  clearCredentials,
+  main,
+};

package/package.json

@@ -1,16 +1,15 @@
 {
   "name": "extract-from-sitemap",
-  "bin": "cli.ts",
-  "version": "0.0.2",
+  "bin": "cli.js",
+  "version": "0.0.4",
   "main": "mod.js",
   "description": "A module and CLI that allows extracting all pages from a sitemap into markdown and a llms.txt, using Parallel.ai APIs.",
   "files": [
     "mod.js",
-    "cli.ts"
+    "cli.js"
   ],
   "license": "MIT",
   "devDependencies": {
-    "@cloudflare/workers-types": "4.20251011.0",
-    "@types/bun": "1.3.0"
+    "@cloudflare/workers-types": "4.20251011.0"
   }
 }