@promptbook/website-crawler 0.82.0 → 0.83.0

package/esm/index.es.js

@@ -26,7 +26,7 @@ var BOOK_LANGUAGE_VERSION = '1.0.0';
  * @generated
  * @see https://github.com/webgptorg/promptbook
  */
-var PROMPTBOOK_ENGINE_VERSION = '0.82.0-3';
+var PROMPTBOOK_ENGINE_VERSION = '0.82.0';
 /**
  * TODO: string_promptbook_version should be constrained to the all versions of Promptbook engine
  * Note: [💞] Ignore a discrepancy between file name and entity name
@@ -3461,15 +3461,15 @@ function $registeredScrapersMessage(availableScrapers) {
 /**
  * Creates unique name for the source
  *
- * @private within the repository
+ * @public exported from `@promptbook/editable`
  */
-function sourceContentToName(sourceContent) {
-    var hash = SHA256(hexEncoder.parse(JSON.stringify(sourceContent)))
+function knowledgeSourceContentToName(knowledgeSourceContent) {
+    var hash = SHA256(hexEncoder.parse(JSON.stringify(knowledgeSourceContent)))
         //    <- TODO: [🥬] Encapsulate sha256 to some private utility function
         .toString( /* hex */)
         .substring(0, 20);
     //    <- TODO: [🥬] Make some system for hashes and ids of promptbook
-    var semanticName = normalizeToKebabCase(sourceContent.substring(0, 20));
+    var semanticName = normalizeToKebabCase(knowledgeSourceContent.substring(0, 20));
     var pieces = ['source', semanticName, hash].filter(function (piece) { return piece !== ''; });
     var name = pieces.join('-').split('--').join('-');
     // <- TODO: Use MAX_FILENAME_LENGTH
@@ -3569,19 +3569,19 @@ var scraperFetch = function (url, init) { return __awaiter(void 0, void 0, void
 function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
     var _a;
     return __awaiter(this, void 0, void 0, function () {
-        var _b, fetch, sourceContent, name, _c, _d, rootDirname, url, response_1, mimeType, filename_1, fileExtension, mimeType;
+        var _b, fetch, knowledgeSourceContent, name, _c, _d, rootDirname, url, response_1, mimeType, filename_1, fileExtension, mimeType;
         return __generator(this, function (_f) {
             switch (_f.label) {
                 case 0:
                     _b = tools.fetch, fetch = _b === void 0 ? scraperFetch : _b;
-                    sourceContent = knowledgeSource.sourceContent;
+                    knowledgeSourceContent = knowledgeSource.knowledgeSourceContent;
                     name = knowledgeSource.name;
                     _c = options || {}, _d = _c.rootDirname, rootDirname = _d === void 0 ? null : _d, _c.isVerbose;
                     if (!name) {
-                        name = sourceContentToName(sourceContent);
+                        name = knowledgeSourceContentToName(knowledgeSourceContent);
                     }
-                    if (!isValidUrl(sourceContent)) return [3 /*break*/, 2];
-                    url = sourceContent;
+                    if (!isValidUrl(knowledgeSourceContent)) return [3 /*break*/, 2];
+                    url = knowledgeSourceContent;
                     return [4 /*yield*/, fetch(url)];
                 case 1:
                     response_1 = _f.sent();
@@ -3627,7 +3627,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
                             },
                         }];
                 case 2:
-                    if (!isValidFilePath(sourceContent)) return [3 /*break*/, 4];
+                    if (!isValidFilePath(knowledgeSourceContent)) return [3 /*break*/, 4];
                     if (tools.fs === undefined) {
                         throw new EnvironmentMismatchError('Can not import file knowledge without filesystem tools');
                         //          <- TODO: [🧠] What is the best error type here`
@@ -3636,13 +3636,13 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
                         throw new EnvironmentMismatchError('Can not import file knowledge in non-file pipeline');
                         //          <- TODO: [🧠] What is the best error type here`
                     }
-                    filename_1 = join(rootDirname, sourceContent).split('\\').join('/');
+                    filename_1 = join(rootDirname, knowledgeSourceContent).split('\\').join('/');
                     fileExtension = getFileExtension(filename_1);
                     mimeType = extensionToMimeType(fileExtension || '');
                     return [4 /*yield*/, isFileExisting(filename_1, tools.fs)];
                 case 3:
                     if (!(_f.sent())) {
-                        throw new NotFoundError(spaceTrim$1(function (block) { return "\n                          Can not make source handler for file which does not exist:\n\n                          File:\n                          ".concat(block(sourceContent), "\n\n                          Full file path:\n                          ").concat(block(filename_1), "\n                      "); }));
+                        throw new NotFoundError(spaceTrim$1(function (block) { return "\n                          Can not make source handler for file which does not exist:\n\n                          File:\n                          ".concat(block(knowledgeSourceContent), "\n\n                          Full file path:\n                          ").concat(block(filename_1), "\n                      "); }));
                     }
                     // TODO: [🧠][😿] Test security file - file is scoped to the project (BUT maybe do this in `filesystemTools`)
                     return [2 /*return*/, {
@@ -3693,7 +3693,7 @@ function makeKnowledgeSourceHandler(knowledgeSource, tools, options) {
                         url: null,
                         mimeType: 'text/markdown',
                         asText: function () {
-                            return knowledgeSource.sourceContent;
+                            return knowledgeSource.knowledgeSourceContent;
                         },
                         asJson: function () {
                             throw new UnexpectedError('Did not expect that `markdownScraper` would need to get the content `asJson`');
@@ -3755,7 +3755,7 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
                                                             partialPieces = __spreadArray([], __read(partialPiecesUnchecked), false);
                                                             return [2 /*return*/, "break"];
                                                         }
-                                                        console.warn(spaceTrim$1(function (block) { return "\n                        Cannot scrape knowledge from source despite the scraper `".concat(scraper.metadata.className, "` supports the mime type \"").concat(sourceHandler.mimeType, "\".\n\n                        The source:\n                        ").concat(block(knowledgeSource.sourceContent
+                                                        console.warn(spaceTrim$1(function (block) { return "\n                        Cannot scrape knowledge from source despite the scraper `".concat(scraper.metadata.className, "` supports the mime type \"").concat(sourceHandler.mimeType, "\".\n\n                        The source:\n                        ").concat(block(knowledgeSource.knowledgeSourceContent
                                                             .split('\n')
                                                             .map(function (line) { return "> ".concat(line); })
                                                             .join('\n')), "\n\n                        ").concat(block($registeredScrapersMessage(scrapers)), "\n\n\n                    "); }));
@@ -3793,7 +3793,7 @@ function prepareKnowledgePieces(knowledgeSources, tools, options) {
                                         return [7 /*endfinally*/];
                                     case 9:
                                         if (partialPieces === null) {
-                                            throw new KnowledgeScrapeError(spaceTrim$1(function (block) { return "\n                        Cannot scrape knowledge\n\n                        The source:\n                        > ".concat(block(knowledgeSource.sourceContent
+                                            throw new KnowledgeScrapeError(spaceTrim$1(function (block) { return "\n                        Cannot scrape knowledge\n\n                        The source:\n                        > ".concat(block(knowledgeSource.knowledgeSourceContent
                                                 .split('\n')
                                                 .map(function (line) { return "> ".concat(line); })
                                                 .join('\n')), "\n\n                        No scraper found for the mime type \"").concat(sourceHandler.mimeType, "\"\n\n                        ").concat(block($registeredScrapersMessage(scrapers)), "\n\n\n                    "); }));