@promptbook/website-crawler 0.100.0 → 0.100.2

package/esm/index.es.js

@@ -4,7 +4,7 @@ import { JSDOM } from 'jsdom';
 import { SHA256 } from 'crypto-js';
 import hexEncoder from 'crypto-js/enc-hex';
 import { mkdir, rm } from 'fs/promises';
-import { basename, join, dirname } from 'path';
+import { basename, join, dirname, isAbsolute } from 'path';
 import parserHtml from 'prettier/parser-html';
 import parserMarkdown from 'prettier/parser-markdown';
 import { format } from 'prettier/standalone';
@@ -30,7 +30,7 @@ const BOOK_LANGUAGE_VERSION = '1.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-const PROMPTBOOK_ENGINE_VERSION = '0.100.0';
+const PROMPTBOOK_ENGINE_VERSION = '0.100.2';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -901,7 +901,7 @@ function removeEmojis(text) {
 }
 
 /**
- * Tests if given string is valid URL.
+ * Tests if given string is valid file path.
  *
  * Note: This does not check if the file exists only if the path is valid
  * @public exported from `@promptbook/utils`
@@ -913,18 +913,25 @@ function isValidFilePath(filename) {
     if (filename.split('\n').length > 1) {
         return false;
     }
-    if (filename.split(' ').length >
-        5 /* <- TODO: [🧠][🈷] Make some better non-arbitrary way how to distinct filenames from informational texts */) {
+    // Normalize slashes early so heuristics can detect path-like inputs
+    const filenameSlashes = filename.replace(/\\/g, '/');
+    // Reject strings that look like sentences (informational text)
+    // Heuristic: contains multiple spaces and ends with a period, or contains typical sentence punctuation
+    // But skip this heuristic if the string looks like a path (contains '/' or starts with a drive letter)
+    if (filename.trim().length > 60 && // long enough to be a sentence
+        /[.!?]/.test(filename) && // contains sentence punctuation
+        filename.split(' ').length > 8 && // has many words
+        !/\/|^[A-Z]:/i.test(filenameSlashes) // do NOT treat as sentence if looks like a path
+    ) {
         return false;
     }
-    const filenameSlashes = filename.split('\\').join('/');
     // Absolute Unix path: /hello.txt
     if (/^(\/)/i.test(filenameSlashes)) {
         // console.log(filename, 'Absolute Unix path: /hello.txt');
         return true;
     }
-    // Absolute Windows path: /hello.txt
-    if (/^([A-Z]{1,2}:\/?)\//i.test(filenameSlashes)) {
+    // Absolute Windows path: C:/ or C:\ (allow spaces and multiple dots in filename)
+    if (/^[A-Z]:\/.+$/i.test(filenameSlashes)) {
         // console.log(filename, 'Absolute Windows path: /hello.txt');
         return true;
     }
@@ -3119,7 +3126,7 @@ const LLM_PROVIDER_PROFILES = {
 };
 /**
  * TODO: Refactor this - each profile must be alongside the provider definition
- * TODO: Unite `AvatarProfileProps` and `ChatParticipant`
+ * TODO: [🕛] Unite `AvatarProfileProps`, `ChatParticipant`, `LlmExecutionTools` +  `LlmToolsMetadata`
  * Note: [💞] Ignore a discrepancy between file name and entity name
  */
 
@@ -3641,9 +3648,15 @@ async function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
     }
     if (isValidUrl(knowledgeSourceContent)) {
         const url = knowledgeSourceContent;
+        if (isVerbose) {
+            console.info(`📄 [1] "${name}" is available at "${url}"`);
+        }
         const response = await fetch(url); // <- TODO: [🧠] Scraping and fetch proxy
         const mimeType = ((_a = response.headers.get('content-type')) === null || _a === void 0 ? void 0 : _a.split(';')[0]) || 'text/html';
         if (tools.fs === undefined || !url.endsWith('.pdf' /* <- TODO: [💵] */)) {
+            if (isVerbose) {
+                console.info(`📄 [2] "${name}" tools.fs is not available or URL is not a PDF.`);
+            }
             return {
                 source: name,
                 filename: null,
@@ -3679,13 +3692,17 @@ async function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
             await tools.fs.mkdir(dirname(join(rootDirname, filepath)), { recursive: true });
         }
         catch (error) {
+            if (isVerbose) {
+                console.info(`📄 [3] "${name}" error creating cache directory`);
+            }
             // Note: If we can't create cache directory, we'll handle it when trying to write the file
             //       This handles read-only filesystems, permission issues, and missing parent directories
-            if (error instanceof Error && (error.message.includes('EROFS') ||
-                error.message.includes('read-only') ||
-                error.message.includes('EACCES') ||
-                error.message.includes('EPERM') ||
-                error.message.includes('ENOENT'))) ;
+            if (error instanceof Error &&
+                (error.message.includes('EROFS') ||
+                    error.message.includes('read-only') ||
+                    error.message.includes('EACCES') ||
+                    error.message.includes('EPERM') ||
+                    error.message.includes('ENOENT'))) ;
             else {
                 // Re-throw other unexpected errors
                 throw error;
@@ -3700,13 +3717,17 @@ async function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
             await tools.fs.writeFile(join(rootDirname, filepath), fileContent);
         }
         catch (error) {
+            if (isVerbose) {
+                console.info(`📄 [4] "${name}" error writing cache file`);
+            }
             // Note: If we can't write to cache, we'll process the file directly from memory
             //       This handles read-only filesystems like Vercel
-            if (error instanceof Error && (error.message.includes('EROFS') ||
-                error.message.includes('read-only') ||
-                error.message.includes('EACCES') ||
-                error.message.includes('EPERM') ||
-                error.message.includes('ENOENT'))) {
+            if (error instanceof Error &&
+                (error.message.includes('EROFS') ||
+                    error.message.includes('read-only') ||
+                    error.message.includes('EACCES') ||
+                    error.message.includes('EPERM') ||
+                    error.message.includes('ENOENT'))) {
                 // Return a handler that works directly with the downloaded content
                 return {
                     source: name,
@@ -3728,6 +3749,9 @@ async function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
         }
         // TODO: [💵] Check the file security
         // TODO: [🧹][🧠] Delete the file after the scraping is done
+        if (isVerbose) {
+            console.info(`📄 [5] "${name}" cached at "${join(rootDirname, filepath)}"`);
+        }
         return makeKnowledgeSourceHandler({ name, knowledgeSourceContent: filepath }, tools, {
             ...options,
             rootDirname,
@@ -3742,7 +3766,12 @@ async function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
             throw new EnvironmentMismatchError('Can not import file knowledge in non-file pipeline');
             //          <- TODO: [🧠] What is the best error type here`
         }
-        const filename = join(rootDirname, knowledgeSourceContent).split('\\').join('/');
+        const filename = isAbsolute(knowledgeSourceContent)
+            ? knowledgeSourceContent
+            : join(rootDirname, knowledgeSourceContent).split('\\').join('/');
+        if (isVerbose) {
+            console.info(`📄 [6] "${name}" is a valid file "${filename}"`);
+        }
         const fileExtension = getFileExtension(filename);
         const mimeType = extensionToMimeType(fileExtension || '');
         if (!(await isFileExisting(filename, tools.fs))) {
@@ -3784,6 +3813,12 @@ async function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
         };
     }
     else {
+        if (isVerbose) {
+            console.info(`📄 [7] "${name}" is just a explicit string text with a knowledge source`);
+            console.info('---');
+            console.info(knowledgeSourceContent);
+            console.info('---');
+        }
         return {
             source: name,
             filename: null,