@nmakarov/cli-toolkit 0.11.3 → 0.14.0

package/dist/index.js

@@ -2502,20 +2502,48 @@ var FileDatabase = class {
   }
   /**
    * Figure out what data to write and which file to use (for pagination)
+   * @param data - Data to write
+   * @param targetFileIndex - Optional index of existing file to overwrite (when customMetadata matches)
+   * @param forceNewFile - If true, always create a new file (when customMetadata provided but no match)
    */
-  figureOutDataAndFileToWrite(data) {
+  figureOutDataAndFileToWrite(data, targetFileIndex = null, forceNewFile = false) {
     let dataToWrite;
     let dataLeftOver;
     const incomingDataType = detectDataType(data);
     if (this.metadata.dataType !== incomingDataType) {
       this.metadata.dataType = incomingDataType;
     }
-    if (this.metadata.files.length === 0) {
+    if (targetFileIndex !== null && targetFileIndex < this.metadata.files.length) {
+      const targetFile = this.metadata.files[targetFileIndex];
+      if (!Array.isArray(data)) {
+        dataToWrite = data;
+        dataLeftOver = null;
+        return { dataToWrite, dataLeftOver, fileName: targetFile.fileName };
+      } else {
+        dataToWrite = data.slice(0, this.pageSize);
+        dataLeftOver = data.slice(this.pageSize);
+        this.lastFileData = dataToWrite;
+        return { dataToWrite, dataLeftOver, fileName: targetFile.fileName };
+      }
+    }
+    let newlyCreatedFileIndex = null;
+    if (forceNewFile) {
+      const filesBeforeCreate = this.metadata.files.length;
+      this.makeNewFile();
+      newlyCreatedFileIndex = filesBeforeCreate;
+      this.logger.debug?.(`[FileDatabase] Creating new file for unique custom metadata combination, fileNumber: ${this.currentFileNumber}`);
+    } else if (this.metadata.files.length === 0) {
       this.makeNewFile();
     }
     const lastFile = this.metadata.files[this.metadata.files.length - 1];
     const lastFileRecordsCount = lastFile.recordsCount;
-    if (!Array.isArray(data)) {
+    if (forceNewFile && newlyCreatedFileIndex !== null) {
+      const newlyCreatedFile = this.metadata.files[newlyCreatedFileIndex];
+      if (newlyCreatedFile && newlyCreatedFile.fileName !== lastFile.fileName) {
+        this.logger.warn?.(`[FileDatabase] Warning: Newly created file ${newlyCreatedFile.fileName} doesn't match last file ${lastFile.fileName}`);
+      }
+    }
+    if (!Array.isArray(data) && !forceNewFile) {
       const lastFileExtension = path3.extname(lastFile.fileName);
       const expectedExtension = `.${getFileExtension(incomingDataType)}`;
       if (lastFileExtension !== expectedExtension) {
@@ -2525,24 +2553,35 @@ var FileDatabase = class {
           lastFile.fileName = `${this.currentFileNumber.toString().padStart(6, "0")}.${getFileExtension(incomingDataType)}`;
         }
       }
+    } else if (!Array.isArray(data) && forceNewFile) {
+      const lastFileExtension = path3.extname(lastFile.fileName);
+      const expectedExtension = `.${getFileExtension(incomingDataType)}`;
+      if (lastFileExtension !== expectedExtension) {
+        lastFile.fileName = `${this.currentFileNumber.toString().padStart(6, "0")}.${getFileExtension(incomingDataType)}`;
+      }
     }
     if (Array.isArray(data)) {
-      if (lastFileRecordsCount < this.pageSize) {
+      if (forceNewFile) {
+        dataToWrite = data.slice(0, this.pageSize);
+        dataLeftOver = data.slice(this.pageSize);
+        this.lastFileData = dataToWrite;
+      } else if (lastFileRecordsCount < this.pageSize) {
         dataToWrite = [...this.lastFileData || [], ...data.slice(0, this.pageSize - lastFileRecordsCount)];
         dataLeftOver = data.slice(this.pageSize - lastFileRecordsCount);
+        this.lastFileData = dataToWrite;
       } else {
         this.makeNewFile();
         dataToWrite = data.slice(0, this.pageSize);
         dataLeftOver = data.slice(this.pageSize);
+        this.lastFileData = dataToWrite;
       }
-      this.lastFileData = dataToWrite;
     } else {
       dataToWrite = data;
       dataLeftOver = null;
     }
     const fileName = this.metadata.files[this.metadata.files.length - 1].fileName;
     this.logger.debug?.(
-      `[FileDatabase] figureOutDataAndFileToWrite: filename=${fileName}, dataToWrite.length=${Array.isArray(dataToWrite) ? dataToWrite.length : "N/A"}, lastFileRecordsCount=${lastFileRecordsCount}`
+      `[FileDatabase] figureOutDataAndFileToWrite: filename=${fileName}, forceNewFile=${forceNewFile}, targetFileIndex=${targetFileIndex}, dataToWrite.length=${Array.isArray(dataToWrite) ? dataToWrite.length : "N/A"}, lastFileRecordsCount=${lastFileRecordsCount}`
     );
     return { dataToWrite, dataLeftOver, fileName };
   }
@@ -2570,7 +2609,7 @@ var FileDatabase = class {
   /**
    * Update metadata after writing data
    */
-  updateMetadata(dataToWrite, fileName) {
+  updateMetadata(dataToWrite, fileName, customMetadata) {
     let currentFile;
     if (fileName) {
       const foundFile = this.metadata.files.find((file) => file.fileName === fileName);
@@ -2585,6 +2624,9 @@ var FileDatabase = class {
     }
     const recordsCount = Array.isArray(dataToWrite) ? dataToWrite.length : 1;
     currentFile.recordsCount = recordsCount;
+    if (customMetadata) {
+      Object.assign(currentFile, customMetadata);
+    }
     const fileIndex = this.metadata.files.indexOf(currentFile);
     if (fileIndex !== -1) {
       this.calculateFileSynopsis(dataToWrite, fileIndex);
@@ -2637,6 +2679,11 @@ var FileDatabase = class {
         } else {
           if (!this.metadata.files.length) {
             this.metadata = await this.figureMetadata(this.currentVersion);
+            if (this.metadata.files && this.metadata.files.length > 0) {
+              this.currentFileNumber = Math.max(...this.metadata.files.map((f) => f.number || 0));
+            } else {
+              this.currentFileNumber = 0;
+            }
           }
         }
       } else {
@@ -2647,16 +2694,22 @@ var FileDatabase = class {
             try {
               const rawData = await fs3.promises.readFile(metadataPath, "utf8");
               this.metadata = JSON.parse(rawData);
+              if (this.metadata.files && this.metadata.files.length > 0) {
+                this.currentFileNumber = Math.max(...this.metadata.files.map((f) => f.number || 0));
+              } else {
+                this.currentFileNumber = 0;
+              }
             } catch (e) {
               this.metadata = this.getDefaultMetadata();
+              this.currentFileNumber = 0;
             }
           } else {
             this.metadata = this.getDefaultMetadata();
-            this.makeNewFile();
+            this.currentFileNumber = 0;
           }
         } else {
           this.metadata = this.getDefaultMetadata();
-          this.makeNewFile();
+          this.currentFileNumber = 0;
         }
       }
     } else if (read) {
@@ -2675,6 +2728,11 @@ var FileDatabase = class {
         }
         if (!this.metadata.files.length) {
           this.metadata = await this.figureMetadata(this.currentVersion);
+          if (this.metadata.files && this.metadata.files.length > 0) {
+            this.currentFileNumber = Math.max(...this.metadata.files.map((f) => f.number || 0));
+          } else {
+            this.currentFileNumber = 0;
+          }
         }
       } else {
         this.currentVersion = null;
@@ -2688,6 +2746,11 @@ var FileDatabase = class {
             try {
               const rawData = await fs3.promises.readFile(metadataPath, "utf8");
               this.metadata = JSON.parse(rawData);
+              if (this.metadata.files && this.metadata.files.length > 0) {
+                this.currentFileNumber = Math.max(...this.metadata.files.map((f) => f.number || 0));
+              } else {
+                this.currentFileNumber = 0;
+              }
             } catch (e) {
               throw new FileDatabaseError(`Failed to read metadata: ${e.message}`);
             }
@@ -2696,6 +2759,11 @@ var FileDatabase = class {
           }
         } else {
           this.metadata = await this.figureMetadataFromVersionFiles("");
+          if (this.metadata.files && this.metadata.files.length > 0) {
+            this.currentFileNumber = Math.max(...this.metadata.files.map((f) => f.number || 0));
+          } else {
+            this.currentFileNumber = 0;
+          }
         }
       }
     }
@@ -2717,14 +2785,44 @@ var FileDatabase = class {
       this.metadata.dataType = incomingDataType;
       this.makeNewFile();
     }
-    let { dataToWrite, dataLeftOver, fileName } = this.figureOutDataAndFileToWrite(data);
+    let targetFileIndex = null;
+    const hasCustomMetadata = options.customMetadata && Object.keys(options.customMetadata).length > 0;
+    if (hasCustomMetadata) {
+      for (let i = 0; i < this.metadata.files.length; i++) {
+        const fileEntry = this.metadata.files[i];
+        const matches = Object.keys(options.customMetadata).every((key) => {
+          return key in fileEntry && fileEntry[key] === options.customMetadata[key];
+        });
+        if (matches) {
+          targetFileIndex = i;
+          this.logger.debug?.(`[FileDatabase] Found existing file with matching custom metadata: ${fileEntry.fileName}, metadata: ${JSON.stringify(options.customMetadata)}`);
+          break;
+        } else {
+          this.logger.debug?.(`[FileDatabase] File ${fileEntry.fileName} does not match custom metadata: ${JSON.stringify(options.customMetadata)}`);
+        }
+      }
+      if (targetFileIndex === null) {
+        this.logger.debug?.(`[FileDatabase] No existing file found with custom metadata: ${JSON.stringify(options.customMetadata)}, will create new file`);
+      }
+    } else {
+      this.logger.debug?.(`[FileDatabase] No custom metadata provided, will create new file`);
+    }
+    if (targetFileIndex !== null) {
+      const targetFile = this.metadata.files[targetFileIndex];
+      this.currentFileNumber = targetFile.number;
+      this.lastFileData = null;
+      this.currentRecord = 0;
+      this.hasReadFirstPage = false;
+    }
+    const forceNewFile = hasCustomMetadata && targetFileIndex === null;
+    let { dataToWrite, dataLeftOver, fileName } = this.figureOutDataAndFileToWrite(data, targetFileIndex, forceNewFile);
     const destPath = this.getDestinationPath(this.currentVersion || void 0);
     await this.safeWrite(path3.join(destPath, fileName), dataToWrite);
-    this.updateMetadata(dataToWrite, fileName);
-    while (dataLeftOver && dataLeftOver.length > 0) {
+    this.updateMetadata(dataToWrite, fileName, options.customMetadata);
+    while (dataLeftOver && dataLeftOver.length > 0 && targetFileIndex === null) {
       const writeContext = this.figureOutDataAndFileToWrite(dataLeftOver);
       await this.safeWrite(path3.join(destPath, writeContext.fileName), writeContext.dataToWrite);
-      this.updateMetadata(writeContext.dataToWrite, writeContext.fileName);
+      this.updateMetadata(writeContext.dataToWrite, writeContext.fileName, options.customMetadata);
       dataLeftOver = writeContext.dataLeftOver;
     }
     this.calculateVersionSynopsis();
@@ -2841,7 +2939,85 @@ var FileDatabase = class {
   getMetadata() {
     return { ...this.metadata };
   }
+  /**
+   * Find data by custom metadata fields
+   * Searches through all versions and files to find entries matching the search criteria
+   * 
+   * @param searchCriteria - Object with field names and values to search for (e.g., { ListingKey: "123", id: "456" })
+   * @returns Array of found entries with their file paths and metadata
+   */
+  async findData(searchCriteria) {
+    const results = [];
+    if (!this.versioned) {
+      await this.prepare({ read: true });
+      const metadata = this.getMetadata();
+      for (const fileEntry of metadata.files) {
+        const matches = Object.keys(searchCriteria).every((key) => {
+          return fileEntry[key] === searchCriteria[key];
+        });
+        if (matches) {
+          const destPath = this.getDestinationPath();
+          const filePath = path3.join(destPath, fileEntry.fileName);
+          const fileData = await fs3.promises.readFile(filePath, "utf8");
+          const data = deserializeData(fileData, metadata.dataType || "json-object");
+          results.push({
+            filePath,
+            fileName: fileEntry.fileName,
+            version: null,
+            metadata: fileEntry,
+            data
+          });
+        }
+      }
+    } else {
+      const versions = await this.getVersions();
+      for (const version of versions) {
+        await this.prepare({ read: true, version });
+        const metadata = this.getMetadata();
+        for (const fileEntry of metadata.files) {
+          const matches = Object.keys(searchCriteria).every((key) => {
+            return fileEntry[key] === searchCriteria[key];
+          });
+          if (matches) {
+            const destPath = this.getDestinationPath(version);
+            const filePath = path3.join(destPath, fileEntry.fileName);
+            const fileData = await fs3.promises.readFile(filePath, "utf8");
+            const data = deserializeData(fileData, metadata.dataType || "json-object");
+            results.push({
+              filePath,
+              fileName: fileEntry.fileName,
+              version,
+              metadata: fileEntry,
+              data
+            });
+          }
+        }
+      }
+    }
+    return results;
+  }
 };
+function listTables(basePath, namespace) {
+  const namespacePath = path3.join(basePath, namespace);
+  if (!fs3.existsSync(namespacePath)) {
+    return [];
+  }
+  try {
+    return fs3.readdirSync(namespacePath, { withFileTypes: true }).filter((dirent) => dirent.isDirectory()).map((dirent) => dirent.name);
+  } catch (error) {
+    return [];
+  }
+}
+function listSources(basePath) {
+  if (!fs3.existsSync(basePath)) {
+    return [];
+  }
+  try {
+    return fs3.readdirSync(basePath, { withFileTypes: true }).filter((dirent) => dirent.isDirectory()).map((dirent) => dirent.name);
+  } catch (error) {
+    return [];
+  }
+}
 function fileDatabaseInit(context, options = {}) {
   return new FileDatabase(context, options);
 }
@@ -2854,6 +3030,10 @@ var Db = class {
   logger;
   queriesLog = [];
   isConnected = false;
+  /**
+   * Constructor - accepts config object
+   * Use dbInit() function to initialize with Context
+   */
   constructor(config2) {
     if (!config2.connectionString) {
       throw new ParamError("Db: connectionString is required");
@@ -3140,6 +3320,91 @@ var Db = class {
     return this.isConnected && this.knexInstance !== null;
   }
 };
+function capitalizeFirstLetter(str) {
+  return str.charAt(0).toUpperCase() + str.slice(1);
+}
+async function dbConnect(context, connectionString, name, dbProfile) {
+  const defs = {
+    testDbConnection: "boolean default true",
+    name: "string",
+    poolMin: "number default 2",
+    poolMax: "number default 10",
+    acquireConnectionTimeout: "number default 10000",
+    sslRejectUnauthorized: "boolean default false"
+  };
+  const paramsConfig = context.params.getAll(defs);
+  const config2 = {
+    connectionString,
+    name: paramsConfig.name || name || "default",
+    testConnection: paramsConfig.testDbConnection,
+    profile: dbProfile ?? false,
+    pool: {
+      min: paramsConfig.poolMin,
+      max: paramsConfig.poolMax
+    },
+    acquireConnectionTimeout: paramsConfig.acquireConnectionTimeout,
+    ssl: {
+      rejectUnauthorized: paramsConfig.sslRejectUnauthorized
+    },
+    logger: context.logger
+  };
+  try {
+    const db = new Db(config2);
+    context.registerCleanup(async () => {
+      await db.disconnect();
+      context.logger.debug(`[Db] instance "${name || connectionString}" destroyed`);
+    });
+    await db.connect();
+    context.logger.debug(`[Db] instance "${name || connectionString}" initialized`);
+    return db;
+  } catch (error) {
+    if (error instanceof ParamError) {
+      throw error;
+    }
+    const errorMsg = error instanceof Error ? error.message : String(error);
+    throw new ParamError(`[Db] connect error: ${errorMsg}`);
+  }
+}
+async function dbFindAndConnect(context, dbNameOrConnectionString) {
+  let dbName;
+  let dbConnectionString;
+  let dbProfile;
+  if (dbNameOrConnectionString) {
+    if (dbNameOrConnectionString.match(/^(postgresql|mysql):\/\/[^\s]+:[^\s]+@[^\s]+:\d+\/[^\s]+$/)) {
+      dbName = void 0;
+      dbConnectionString = dbNameOrConnectionString;
+    } else {
+      dbName = dbNameOrConnectionString;
+    }
+  } else {
+    const defs = {
+      dbName: "string",
+      dbConnectionString: "string",
+      dbProfile: "boolean default false"
+    };
+    const paramsConfig = context.params.getAll(defs);
+    dbName = paramsConfig.dbName;
+    dbConnectionString = paramsConfig.dbConnectionString;
+    dbProfile = paramsConfig.dbProfile;
+  }
+  if (!dbName && !dbConnectionString) {
+    throw new ParamError("Db: either dbName or dbConnectionString must be specified");
+  }
+  if (dbName) {
+    const paramName = `dbConnectionString${capitalizeFirstLetter(dbName)}`;
+    dbConnectionString = await context.params.get(paramName, "string");
+    if (!dbConnectionString) {
+      throw new ParamError(
+        `Db: cannot find dbConnectionString for dbName="${dbName}" (looked for param "${paramName}")`
+      );
+    }
+  }
+  const db = await dbConnect(context, dbConnectionString, dbName, dbProfile);
+  return db;
+}
+async function dbInit(context, dbNameOrConnectionString) {
+  return await dbFindAndConnect(context, dbNameOrConnectionString);
+}
 
 // src/logger/index.ts
 import chalk from "chalk";
@@ -3534,6 +3799,9 @@ export {
   buildBreadcrumb,
   buildDetailBreadcrumb,
   buildFooter,
+  dbConnect,
+  dbFindAndConnect,
+  dbInit,
   defaultFileSynopsisFunction,
   defaultVersionSynopsisFunction,
   fileDatabaseInit,
@@ -3541,6 +3809,8 @@ export {
   createElement2 as h,
   joiEdateType,
   joiStringArrayType,
+  listSources,
+  listTables,
   load,
   organizeFooterMessages,
   setupContext,