@promptbook/website-crawler 0.72.0-33 → 0.72.0-34

package/README.md

@@ -52,15 +52,15 @@ Rest of the documentation is common for **entire promptbook ecosystem**:
 
 
 
-If you have a simple, single prompt for ChatGPT, GPT-4, Anthropic Claude, Google Gemini, Llama 2, or whatever, it doesn't matter how you integrate it. Whether it's calling a REST API directly, using the SDK, hardcoding the prompt into the source code, or importing a text file, the process remains the same.
+If you have a simple, single prompt for ChatGPT, GPT-4, Anthropic Claude, Google Gemini, Llama 3, or whatever, it doesn't matter how you integrate it. Whether it's calling a REST API directly, using the SDK, hardcoding the prompt into the source code, or importing a text file, the process remains the same.
 
-But often you will struggle with the limitations of LLMs, such as hallucinations, off-topic responses, poor quality output, language drift, word repetition repetition repetition repetition or misuse, lack of context, or just plain w𝒆𝐢rd responses. When this happens, you generally have three options:
+But often you will struggle with the **limitations of LLMs**, such as **hallucinations, off-topic responses, poor quality output, language and prompt drift, word repetition repetition repetition repetition or misuse, lack of context, or just plain w𝒆𝐢rd responses**. When this happens, you generally have three options:
 
 1. **Fine-tune** the model to your specifications or even train your own.
 2. **Prompt-engineer** the prompt to the best shape you can achieve.
 3. Orchestrate **multiple prompts** in a [pipeline](https://github.com/webgptorg/promptbook/discussions/64) to get the best result.
 
-In all of these situations, but especially in 3., the Promptbook library can make your life easier.
+In all of these situations, but especially in 3., the **✨ Promptbook can make your life waaaaaaaaaay easier**.
 
 -   [**Separates concerns**](https://github.com/webgptorg/promptbook/discussions/32) between prompt-engineer and programmer, between code files and prompt files, and between prompts and their execution logic.
 -   Establishes a [**common format `.ptbk.md`**](https://github.com/webgptorg/promptbook/discussions/85) that can be used to describe your prompt business logic without having to write code or deal with the technicalities of LLMs.

package/esm/index.es.js

@@ -1,7 +1,6 @@
 import spaceTrim$1, { spaceTrim } from 'spacetrim';
 import { Readability } from '@mozilla/readability';
 import { JSDOM } from 'jsdom';
-import { Converter } from 'showdown';
 import { SHA256 } from 'crypto-js';
 import hexEncoder from 'crypto-js/enc-hex';
 import { mkdir, rm } from 'fs/promises';
@@ -11,12 +10,13 @@ import parserHtml from 'prettier/parser-html';
 import { forTime } from 'waitasecond';
 import { lookup } from 'mime-types';
 import { unparse, parse } from 'papaparse';
+import { Converter } from 'showdown';
 
 // ⚠️ WARNING: This code has been generated so that any manual changes will be overwritten
 /**
  * The version of the Promptbook library
  */
-var PROMPTBOOK_VERSION = '0.72.0-32';
+var PROMPTBOOK_VERSION = '0.72.0-33';
 // TODO: [main] !!!! List here all the versions and annotate + put into script
 
 /*! *****************************************************************************
@@ -679,44 +679,6 @@ var KnowledgeScrapeError = /** @class */ (function (_super) {
     return KnowledgeScrapeError;
 }(Error));
 
-/**
- * Just says that the variable is not used but should be kept
- * No side effects.
- *
- * Note: It can be usefull for:
- *
- * 1) Suppressing eager optimization of unused imports
- * 2) Suppressing eslint errors of unused variables in the tests
- * 3) Keeping the type of the variable for type testing
- *
- * @param value any values
- * @returns void
- * @private within the repository
- */
-function keepUnused() {
-    var valuesToKeep = [];
-    for (var _i = 0; _i < arguments.length; _i++) {
-        valuesToKeep[_i] = arguments[_i];
-    }
-}
-
-/**
- * Just marks a place of place where should be something implemented
- * No side effects.
- *
- * Note: It can be usefull suppressing eslint errors of unused variables
- *
- * @param value any values
- * @returns void
- * @private within the repository
- */
-function TODO_USE() {
-    var value = [];
-    for (var _i = 0; _i < arguments.length; _i++) {
-        value[_i] = arguments[_i];
-    }
-}
-
 /**
  * Tests if given string is valid URL.
  *
@@ -1141,6 +1103,23 @@ function nameToSubfolderPath(name) {
     return [name.substr(0, 1).toLowerCase(), name.substr(1, 1).toLowerCase()];
 }
 
+/**
+ * Just marks a place of place where should be something implemented
+ * No side effects.
+ *
+ * Note: It can be usefull suppressing eslint errors of unused variables
+ *
+ * @param value any values
+ * @returns void
+ * @private within the repository
+ */
+function TODO_USE() {
+    var value = [];
+    for (var _i = 0; _i < arguments.length; _i++) {
+        value[_i] = arguments[_i];
+    }
+}
+
 /**
  * Create a filename for intermediate cache for scrapers
  *
@@ -4301,6 +4280,27 @@ function extractJsonBlock(markdown) {
  * TODO: [🏢] Make this logic part of `JsonFormatDefinition` or `isValidJsonString`
  */
 
+/**
+ * Just says that the variable is not used but should be kept
+ * No side effects.
+ *
+ * Note: It can be usefull for:
+ *
+ * 1) Suppressing eager optimization of unused imports
+ * 2) Suppressing eslint errors of unused variables in the tests
+ * 3) Keeping the type of the variable for type testing
+ *
+ * @param value any values
+ * @returns void
+ * @private within the repository
+ */
+function keepUnused() {
+    var valuesToKeep = [];
+    for (var _i = 0; _i < arguments.length; _i++) {
+        valuesToKeep[_i] = arguments[_i];
+    }
+}
+
 /**
  * Replaces parameters in template with values from parameters object
  *
@@ -5826,25 +5826,6 @@ var WebsiteScraper = /** @class */ (function () {
         this.options = options;
         this.markdownScraper = new MarkdownScraper(tools, options);
         this.showdownConverter = createShowdownConverter();
-        /**/
-        // TODO:  [🏄] !!!!!! Remove or describe why it is here
-        TODO_USE(Readability);
-        TODO_USE(Converter);
-        TODO_USE(JSDOM);
-        TODO_USE(createShowdownConverter);
-        /**/
-        /**/
-        var jsdom = new JSDOM();
-        var reader = new Readability(jsdom.window.document);
-        keepUnused(reader);
-        /**/
-        /**/
-        keepUnused(DEFAULT_INTERMEDIATE_FILES_STRATEGY, DEFAULT_IS_VERBOSE, DEFAULT_SCRAPE_CACHE_DIRNAME);
-        keepUnused(EnvironmentMismatchError);
-        keepUnused(KnowledgeScrapeError);
-        keepUnused(getScraperIntermediateSource);
-        keepUnused();
-        /**/
     }
     Object.defineProperty(WebsiteScraper.prototype, "metadata", {
         /**
@@ -5870,8 +5851,6 @@ var WebsiteScraper = /** @class */ (function () {
                 switch (_g.label) {
                     case 0:
                         _a = this.options, _b = _a.rootDirname, rootDirname = _b === void 0 ? process.cwd() : _b, _c = _a.cacheDirname, cacheDirname = _c === void 0 ? DEFAULT_SCRAPE_CACHE_DIRNAME : _c, _d = _a.intermediateFilesStrategy, intermediateFilesStrategy = _d === void 0 ? DEFAULT_INTERMEDIATE_FILES_STRATEGY : _d, _e = _a.isVerbose, isVerbose = _e === void 0 ? DEFAULT_IS_VERBOSE : _e;
-                        // [🏄] !!!!!!!
-                        keepUnused(rootDirname, cacheDirname, intermediateFilesStrategy, isVerbose);
                         if (source.url === null) {
                             throw new KnowledgeScrapeError('Website scraper requires URL');
                         }
@@ -5882,11 +5861,8 @@ var WebsiteScraper = /** @class */ (function () {
                         return [4 /*yield*/, source.asText()];
                     case 1:
                         jsdom = new (_f.apply(JSDOM, [void 0, _g.sent(), {
-                                // <- TODO: !!!!!!! Problem with build is probbably in `new JSDOM();`
                                 url: source.url,
                             }]))();
-                        // [🏄] !!!!!!!
-                        keepUnused(jsdom);
                         reader = new Readability(jsdom.window.document);
                         article = reader.parse();
                         html = (article === null || article === void 0 ? void 0 : article.content) || (article === null || article === void 0 ? void 0 : article.textContent) || jsdom.window.document.body.innerHTML;