@nmakarov/cli-toolkit 0.11.4 → 0.14.0

package/dist/filedatabase.cjs

@@ -34,7 +34,9 @@ __export(filedatabase_exports, {
   FileDatabaseError: () => FileDatabaseError,
   defaultFileSynopsisFunction: () => defaultFileSynopsisFunction,
   defaultVersionSynopsisFunction: () => defaultVersionSynopsisFunction,
-  fileDatabaseInit: () => fileDatabaseInit
+  fileDatabaseInit: () => fileDatabaseInit,
+  listSources: () => listSources,
+  listTables: () => listTables
 });
 module.exports = __toCommonJS(filedatabase_exports);
 
@@ -650,20 +652,48 @@ var FileDatabase = class {
   }
   /**
    * Figure out what data to write and which file to use (for pagination)
+   * @param data - Data to write
+   * @param targetFileIndex - Optional index of existing file to overwrite (when customMetadata matches)
+   * @param forceNewFile - If true, always create a new file (when customMetadata provided but no match)
    */
-  figureOutDataAndFileToWrite(data) {
+  figureOutDataAndFileToWrite(data, targetFileIndex = null, forceNewFile = false) {
     let dataToWrite;
     let dataLeftOver;
     const incomingDataType = detectDataType(data);
     if (this.metadata.dataType !== incomingDataType) {
       this.metadata.dataType = incomingDataType;
     }
-    if (this.metadata.files.length === 0) {
+    if (targetFileIndex !== null && targetFileIndex < this.metadata.files.length) {
+      const targetFile = this.metadata.files[targetFileIndex];
+      if (!Array.isArray(data)) {
+        dataToWrite = data;
+        dataLeftOver = null;
+        return { dataToWrite, dataLeftOver, fileName: targetFile.fileName };
+      } else {
+        dataToWrite = data.slice(0, this.pageSize);
+        dataLeftOver = data.slice(this.pageSize);
+        this.lastFileData = dataToWrite;
+        return { dataToWrite, dataLeftOver, fileName: targetFile.fileName };
+      }
+    }
+    let newlyCreatedFileIndex = null;
+    if (forceNewFile) {
+      const filesBeforeCreate = this.metadata.files.length;
+      this.makeNewFile();
+      newlyCreatedFileIndex = filesBeforeCreate;
+      this.logger.debug?.(`[FileDatabase] Creating new file for unique custom metadata combination, fileNumber: ${this.currentFileNumber}`);
+    } else if (this.metadata.files.length === 0) {
       this.makeNewFile();
     }
     const lastFile = this.metadata.files[this.metadata.files.length - 1];
     const lastFileRecordsCount = lastFile.recordsCount;
-    if (!Array.isArray(data)) {
+    if (forceNewFile && newlyCreatedFileIndex !== null) {
+      const newlyCreatedFile = this.metadata.files[newlyCreatedFileIndex];
+      if (newlyCreatedFile && newlyCreatedFile.fileName !== lastFile.fileName) {
+        this.logger.warn?.(`[FileDatabase] Warning: Newly created file ${newlyCreatedFile.fileName} doesn't match last file ${lastFile.fileName}`);
+      }
+    }
+    if (!Array.isArray(data) && !forceNewFile) {
       const lastFileExtension = import_path3.default.extname(lastFile.fileName);
       const expectedExtension = `.${getFileExtension(incomingDataType)}`;
       if (lastFileExtension !== expectedExtension) {
@@ -673,24 +703,35 @@ var FileDatabase = class {
           lastFile.fileName = `${this.currentFileNumber.toString().padStart(6, "0")}.${getFileExtension(incomingDataType)}`;
         }
       }
+    } else if (!Array.isArray(data) && forceNewFile) {
+      const lastFileExtension = import_path3.default.extname(lastFile.fileName);
+      const expectedExtension = `.${getFileExtension(incomingDataType)}`;
+      if (lastFileExtension !== expectedExtension) {
+        lastFile.fileName = `${this.currentFileNumber.toString().padStart(6, "0")}.${getFileExtension(incomingDataType)}`;
+      }
     }
     if (Array.isArray(data)) {
-      if (lastFileRecordsCount < this.pageSize) {
+      if (forceNewFile) {
+        dataToWrite = data.slice(0, this.pageSize);
+        dataLeftOver = data.slice(this.pageSize);
+        this.lastFileData = dataToWrite;
+      } else if (lastFileRecordsCount < this.pageSize) {
         dataToWrite = [...this.lastFileData || [], ...data.slice(0, this.pageSize - lastFileRecordsCount)];
         dataLeftOver = data.slice(this.pageSize - lastFileRecordsCount);
+        this.lastFileData = dataToWrite;
       } else {
         this.makeNewFile();
         dataToWrite = data.slice(0, this.pageSize);
         dataLeftOver = data.slice(this.pageSize);
+        this.lastFileData = dataToWrite;
       }
-      this.lastFileData = dataToWrite;
     } else {
       dataToWrite = data;
       dataLeftOver = null;
     }
     const fileName = this.metadata.files[this.metadata.files.length - 1].fileName;
     this.logger.debug?.(
-      `[FileDatabase] figureOutDataAndFileToWrite: filename=${fileName}, dataToWrite.length=${Array.isArray(dataToWrite) ? dataToWrite.length : "N/A"}, lastFileRecordsCount=${lastFileRecordsCount}`
+      `[FileDatabase] figureOutDataAndFileToWrite: filename=${fileName}, forceNewFile=${forceNewFile}, targetFileIndex=${targetFileIndex}, dataToWrite.length=${Array.isArray(dataToWrite) ? dataToWrite.length : "N/A"}, lastFileRecordsCount=${lastFileRecordsCount}`
     );
     return { dataToWrite, dataLeftOver, fileName };
   }
@@ -718,7 +759,7 @@ var FileDatabase = class {
   /**
    * Update metadata after writing data
    */
-  updateMetadata(dataToWrite, fileName) {
+  updateMetadata(dataToWrite, fileName, customMetadata) {
     let currentFile;
     if (fileName) {
       const foundFile = this.metadata.files.find((file) => file.fileName === fileName);
@@ -733,6 +774,9 @@ var FileDatabase = class {
     }
     const recordsCount = Array.isArray(dataToWrite) ? dataToWrite.length : 1;
     currentFile.recordsCount = recordsCount;
+    if (customMetadata) {
+      Object.assign(currentFile, customMetadata);
+    }
     const fileIndex = this.metadata.files.indexOf(currentFile);
     if (fileIndex !== -1) {
       this.calculateFileSynopsis(dataToWrite, fileIndex);
@@ -785,6 +829,11 @@ var FileDatabase = class {
         } else {
           if (!this.metadata.files.length) {
             this.metadata = await this.figureMetadata(this.currentVersion);
+            if (this.metadata.files && this.metadata.files.length > 0) {
+              this.currentFileNumber = Math.max(...this.metadata.files.map((f) => f.number || 0));
+            } else {
+              this.currentFileNumber = 0;
+            }
           }
         }
       } else {
@@ -795,16 +844,22 @@ var FileDatabase = class {
             try {
               const rawData = await import_fs3.default.promises.readFile(metadataPath, "utf8");
               this.metadata = JSON.parse(rawData);
+              if (this.metadata.files && this.metadata.files.length > 0) {
+                this.currentFileNumber = Math.max(...this.metadata.files.map((f) => f.number || 0));
+              } else {
+                this.currentFileNumber = 0;
+              }
             } catch (e) {
               this.metadata = this.getDefaultMetadata();
+              this.currentFileNumber = 0;
             }
           } else {
             this.metadata = this.getDefaultMetadata();
-            this.makeNewFile();
+            this.currentFileNumber = 0;
           }
         } else {
           this.metadata = this.getDefaultMetadata();
-          this.makeNewFile();
+          this.currentFileNumber = 0;
         }
       }
     } else if (read) {
@@ -823,6 +878,11 @@ var FileDatabase = class {
         }
         if (!this.metadata.files.length) {
           this.metadata = await this.figureMetadata(this.currentVersion);
+          if (this.metadata.files && this.metadata.files.length > 0) {
+            this.currentFileNumber = Math.max(...this.metadata.files.map((f) => f.number || 0));
+          } else {
+            this.currentFileNumber = 0;
+          }
         }
       } else {
         this.currentVersion = null;
@@ -836,6 +896,11 @@ var FileDatabase = class {
             try {
               const rawData = await import_fs3.default.promises.readFile(metadataPath, "utf8");
               this.metadata = JSON.parse(rawData);
+              if (this.metadata.files && this.metadata.files.length > 0) {
+                this.currentFileNumber = Math.max(...this.metadata.files.map((f) => f.number || 0));
+              } else {
+                this.currentFileNumber = 0;
+              }
             } catch (e) {
               throw new FileDatabaseError(`Failed to read metadata: ${e.message}`);
             }
@@ -844,6 +909,11 @@ var FileDatabase = class {
           }
         } else {
           this.metadata = await this.figureMetadataFromVersionFiles("");
+          if (this.metadata.files && this.metadata.files.length > 0) {
+            this.currentFileNumber = Math.max(...this.metadata.files.map((f) => f.number || 0));
+          } else {
+            this.currentFileNumber = 0;
+          }
         }
       }
     }
@@ -865,14 +935,44 @@ var FileDatabase = class {
       this.metadata.dataType = incomingDataType;
       this.makeNewFile();
     }
-    let { dataToWrite, dataLeftOver, fileName } = this.figureOutDataAndFileToWrite(data);
+    let targetFileIndex = null;
+    const hasCustomMetadata = options.customMetadata && Object.keys(options.customMetadata).length > 0;
+    if (hasCustomMetadata) {
+      for (let i = 0; i < this.metadata.files.length; i++) {
+        const fileEntry = this.metadata.files[i];
+        const matches = Object.keys(options.customMetadata).every((key) => {
+          return key in fileEntry && fileEntry[key] === options.customMetadata[key];
+        });
+        if (matches) {
+          targetFileIndex = i;
+          this.logger.debug?.(`[FileDatabase] Found existing file with matching custom metadata: ${fileEntry.fileName}, metadata: ${JSON.stringify(options.customMetadata)}`);
+          break;
+        } else {
+          this.logger.debug?.(`[FileDatabase] File ${fileEntry.fileName} does not match custom metadata: ${JSON.stringify(options.customMetadata)}`);
+        }
+      }
+      if (targetFileIndex === null) {
+        this.logger.debug?.(`[FileDatabase] No existing file found with custom metadata: ${JSON.stringify(options.customMetadata)}, will create new file`);
+      }
+    } else {
+      this.logger.debug?.(`[FileDatabase] No custom metadata provided, will create new file`);
+    }
+    if (targetFileIndex !== null) {
+      const targetFile = this.metadata.files[targetFileIndex];
+      this.currentFileNumber = targetFile.number;
+      this.lastFileData = null;
+      this.currentRecord = 0;
+      this.hasReadFirstPage = false;
+    }
+    const forceNewFile = hasCustomMetadata && targetFileIndex === null;
+    let { dataToWrite, dataLeftOver, fileName } = this.figureOutDataAndFileToWrite(data, targetFileIndex, forceNewFile);
     const destPath = this.getDestinationPath(this.currentVersion || void 0);
     await this.safeWrite(import_path3.default.join(destPath, fileName), dataToWrite);
-    this.updateMetadata(dataToWrite, fileName);
-    while (dataLeftOver && dataLeftOver.length > 0) {
+    this.updateMetadata(dataToWrite, fileName, options.customMetadata);
+    while (dataLeftOver && dataLeftOver.length > 0 && targetFileIndex === null) {
       const writeContext = this.figureOutDataAndFileToWrite(dataLeftOver);
       await this.safeWrite(import_path3.default.join(destPath, writeContext.fileName), writeContext.dataToWrite);
-      this.updateMetadata(writeContext.dataToWrite, writeContext.fileName);
+      this.updateMetadata(writeContext.dataToWrite, writeContext.fileName, options.customMetadata);
       dataLeftOver = writeContext.dataLeftOver;
     }
     this.calculateVersionSynopsis();
@@ -989,7 +1089,85 @@ var FileDatabase = class {
   getMetadata() {
     return { ...this.metadata };
   }
+  /**
+   * Find data by custom metadata fields
+   * Searches through all versions and files to find entries matching the search criteria
+   * 
+   * @param searchCriteria - Object with field names and values to search for (e.g., { ListingKey: "123", id: "456" })
+   * @returns Array of found entries with their file paths and metadata
+   */
+  async findData(searchCriteria) {
+    const results = [];
+    if (!this.versioned) {
+      await this.prepare({ read: true });
+      const metadata = this.getMetadata();
+      for (const fileEntry of metadata.files) {
+        const matches = Object.keys(searchCriteria).every((key) => {
+          return fileEntry[key] === searchCriteria[key];
+        });
+        if (matches) {
+          const destPath = this.getDestinationPath();
+          const filePath = import_path3.default.join(destPath, fileEntry.fileName);
+          const fileData = await import_fs3.default.promises.readFile(filePath, "utf8");
+          const data = deserializeData(fileData, metadata.dataType || "json-object");
+          results.push({
+            filePath,
+            fileName: fileEntry.fileName,
+            version: null,
+            metadata: fileEntry,
+            data
+          });
+        }
+      }
+    } else {
+      const versions = await this.getVersions();
+      for (const version of versions) {
+        await this.prepare({ read: true, version });
+        const metadata = this.getMetadata();
+        for (const fileEntry of metadata.files) {
+          const matches = Object.keys(searchCriteria).every((key) => {
+            return fileEntry[key] === searchCriteria[key];
+          });
+          if (matches) {
+            const destPath = this.getDestinationPath(version);
+            const filePath = import_path3.default.join(destPath, fileEntry.fileName);
+            const fileData = await import_fs3.default.promises.readFile(filePath, "utf8");
+            const data = deserializeData(fileData, metadata.dataType || "json-object");
+            results.push({
+              filePath,
+              fileName: fileEntry.fileName,
+              version,
+              metadata: fileEntry,
+              data
+            });
+          }
+        }
+      }
+    }
+    return results;
+  }
 };
+function listTables(basePath, namespace) {
+  const namespacePath = import_path3.default.join(basePath, namespace);
+  if (!import_fs3.default.existsSync(namespacePath)) {
+    return [];
+  }
+  try {
+    return import_fs3.default.readdirSync(namespacePath, { withFileTypes: true }).filter((dirent) => dirent.isDirectory()).map((dirent) => dirent.name);
+  } catch (error) {
+    return [];
+  }
+}
+function listSources(basePath) {
+  if (!import_fs3.default.existsSync(basePath)) {
+    return [];
+  }
+  try {
+    return import_fs3.default.readdirSync(basePath, { withFileTypes: true }).filter((dirent) => dirent.isDirectory()).map((dirent) => dirent.name);
+  } catch (error) {
+    return [];
+  }
+}
 function fileDatabaseInit(context, options = {}) {
   return new FileDatabase(context, options);
 }
@@ -999,6 +1177,8 @@ function fileDatabaseInit(context, options = {}) {
   FileDatabaseError,
   defaultFileSynopsisFunction,
   defaultVersionSynopsisFunction,
-  fileDatabaseInit
+  fileDatabaseInit,
+  listSources,
+  listTables
 });
 //# sourceMappingURL=filedatabase.cjs.map