inibase 3.6.0 → 3.7.0

package/dist/file.d.ts

@@ -88,6 +88,20 @@ export declare function get(filePath: string, lineNumbers: undefined | number |
     Record<number, string | number | boolean | null | undefined | (string | number | boolean | (string | number | boolean)[] | null)[]> | null,
     number
 ]>;
+/**
+ * Reads only the requested line numbers of one column file and decodes each
+ * with `field`. Unlike the shell `sed -n` reader this stops as soon as the last
+ * requested line has been seen, so a point lookup never scans the column to
+ * EOF, and it decodes each column independently (no tab-delimited `paste`
+ * alignment to break on values that contain a literal tab). Compression is
+ * handled transparently: `.gz` columns are gunzipped on the fly.
+ *
+ * Every resource (readline, gunzip/source stream, file handle) is released on
+ * both the normal and the early-exit path.
+ */
+export declare const getLines: (filePath: string, lineNumbers: number[], field?: Field & {
+    databasePath?: string;
+}) => Promise<Record<number, ReturnType<typeof decode>>>;
 /**
  * Asynchronously replaces specific lines in a file based on the provided replacements map or string.
  *
@@ -146,7 +160,7 @@ export declare const remove: (filePath: string, linesToDelete: number | number[]
  */
 export declare const search: (filePath: string, operator: ComparisonOperator | ComparisonOperator[], comparedAtValue: string | number | boolean | null | (string | number | boolean | null)[], logicalOperator?: "and" | "or", searchIn?: Set<number>, field?: Field & {
     databasePath?: string;
-}, limit?: number, offset?: number, readWholeFile?: boolean) => Promise<[Record<number, string | number | boolean | null | undefined | (string | number | boolean | null)[]> | null, number, Set<number> | null]>;
+}, limit?: number, offset?: number, readWholeFile?: boolean, earlyStop?: boolean) => Promise<[Record<number, string | number | boolean | null | undefined | (string | number | boolean | null)[]> | null, number, Set<number> | null]>;
 export declare const sum: (fp: string, ln?: number | number[]) => Promise<number>;
 export declare const avg: (fp: string, ln?: number | number[]) => Promise<number>;
 export declare const min: (fp: string, ln?: number | number[]) => Promise<number>;

package/dist/file.js

@@ -343,6 +343,16 @@ export function escapeShellPath(filePath) {
  */
 let tempSeq = 0;
 const tempPathFor = (filePath) => filePath.replace(/([^/]+)\/?$/, `.tmp/$1.${process.pid}-${(tempSeq++).toString(36)}`);
+/**
+ * Cleanup-and-rethrow for a failed column write. A partial temp file must never
+ * be left behind, and the caller must observe the failure so it can abort the
+ * operation instead of publishing a column that is missing a line — which would
+ * desync the pagination row count from the column files.
+ */
+const failWrite = async (fileTempPath, error) => {
+    await unlink(fileTempPath).catch(() => { });
+    throw error;
+};
 const _pipeline = async (filePath, rl, writeStream, transform) => {
     if (filePath.endsWith(".gz"))
         await pipeline(rl, transform, createGzip(), writeStream);
@@ -361,6 +371,30 @@ const createReadLineInternface = (filePath, fileHandle) => createInterface({
         : fileHandle.createReadStream(),
     crlfDelay: Number.POSITIVE_INFINITY,
 });
+/**
+ * Like {@link createReadLineInternface} but also hands back a disposer that
+ * tears down the readline interface and both the source and (for `.gz`
+ * columns) gunzip streams. Sparse readers call the disposer in their `finally`
+ * so an early `break` — the whole point of a point lookup — cannot leave a read
+ * stream (and its libuv handle) dangling, which would keep the process alive.
+ */
+const openReadline = (filePath, fileHandle) => {
+    const source = fileHandle.createReadStream();
+    const input = filePath.endsWith(".gz") ? source.pipe(createGunzip()) : source;
+    const rl = createInterface({
+        input,
+        crlfDelay: Number.POSITIVE_INFINITY,
+    });
+    return {
+        rl,
+        destroy: () => {
+            rl.close();
+            input.destroy();
+            if (input !== source)
+                source.destroy();
+        },
+    };
+};
 /**
  * Checks if a file or directory exists at the specified path.
  *
@@ -619,6 +653,45 @@ export async function get(filePath, lineNumbers, field, readWholeFile = false) {
         await fileHandle?.close();
     }
 }
+/**
+ * Reads only the requested line numbers of one column file and decodes each
+ * with `field`. Unlike the shell `sed -n` reader this stops as soon as the last
+ * requested line has been seen, so a point lookup never scans the column to
+ * EOF, and it decodes each column independently (no tab-delimited `paste`
+ * alignment to break on values that contain a literal tab). Compression is
+ * handled transparently: `.gz` columns are gunzipped on the fly.
+ *
+ * Every resource (readline, gunzip/source stream, file handle) is released on
+ * both the normal and the early-exit path.
+ */
+export const getLines = async (filePath, lineNumbers, field) => {
+    const lines = {};
+    if (!lineNumbers.length)
+        return lines;
+    const wanted = new Set(lineNumbers);
+    const last = Math.max(...lineNumbers);
+    const config = field ?? {
+        key: "BLABLA",
+        type: "string",
+    };
+    const fileHandle = await open(filePath, "r");
+    const { rl, destroy } = openReadline(filePath, fileHandle);
+    try {
+        let linesCount = 0;
+        for await (const line of rl) {
+            linesCount++;
+            if (wanted.has(linesCount))
+                lines[linesCount] = decode(line, config);
+            if (linesCount >= last)
+                break;
+        }
+        return lines;
+    }
+    finally {
+        destroy();
+        await fileHandle.close().catch(() => { });
+    }
+};
 /**
  * Asynchronously replaces specific lines in a file based on the provided replacements map or string.
  *
@@ -675,8 +748,8 @@ export const replace = async (filePath, replacements, totalItems) => {
                 }));
                 return [fileTempPath, filePath];
             }
-            catch {
-                return [fileTempPath, null];
+            catch (error) {
+                return failWrite(fileTempPath, error);
             }
             finally {
                 // Ensure that file handles are closed, even if an error occurred
@@ -695,8 +768,8 @@ export const replace = async (filePath, replacements, totalItems) => {
                 await exec(command);
                 return [fileTempPath, filePath];
             }
-            catch {
-                return [fileTempPath, null];
+            catch (error) {
+                return failWrite(fileTempPath, error);
             }
         }
     }
@@ -736,8 +809,8 @@ export const replace = async (filePath, replacements, totalItems) => {
             }
             return [fileTempPath, filePath];
         }
-        catch {
-            return [fileTempPath, null];
+        catch (error) {
+            return failWrite(fileTempPath, error);
         }
     }
     return [];
@@ -759,10 +832,12 @@ export const append = async (filePath, data) => {
                 await appendFile(fileTempPath, `${Array.isArray(data) ? data.join("\n") : data}\n`);
             }
             else {
-                const escapedFileTempPath = escapeShellPath(fileTempPath);
-                await exec(`echo '${(Array.isArray(data) ? data.join("\n") : data)
-                    .toString()
-                    .replace(/'/g, "\\'")}' | gzip - >> ${escapedFileTempPath}`);
+                // Append a complete gzip member. Readers gunzip every member in
+                // sequence (multi-member stream), and this avoids the shell's
+                // argument-length limits and single-quote escaping the old
+                // `echo … | gzip - >> file` pipeline suffered from.
+                const text = `${Array.isArray(data) ? data.join("\n") : data}\n`;
+                await appendFile(fileTempPath, await gzip(text));
             }
         }
         else
@@ -775,8 +850,8 @@ export const append = async (filePath, data) => {
                 : `${Array.isArray(data) ? data.join("\n") : data}\n`);
         return [fileTempPath, filePath];
     }
-    catch {
-        return [fileTempPath, null];
+    catch (error) {
+        return failWrite(fileTempPath, error);
     }
 };
 /**
@@ -808,8 +883,8 @@ export const prepend = async (filePath, data) => {
                     },
                 }));
             }
-            catch {
-                return [fileTempPath, null];
+            catch (error) {
+                return failWrite(fileTempPath, error);
             }
             finally {
                 // Ensure that file handles are closed, even if an error occurred
@@ -826,8 +901,8 @@ export const prepend = async (filePath, data) => {
                 const escapedFileChildTempPath = escapeShellPath(fileChildTempPath);
                 await exec(`cat ${escapedFileChildTempPath} ${escapedFilePath} > ${escapedFileTempPath}`);
             }
-            catch {
-                return [fileTempPath, null];
+            catch (error) {
+                return failWrite(fileTempPath, error);
             }
             finally {
                 await unlink(fileChildTempPath);
@@ -843,8 +918,8 @@ export const prepend = async (filePath, data) => {
                 ? await gzip(`${Array.isArray(data) ? data.join("\n") : data}\n`)
                 : `${Array.isArray(data) ? data.join("\n") : data}\n`);
         }
-        catch {
-            return [fileTempPath, null];
+        catch (error) {
+            return failWrite(fileTempPath, error);
         }
     }
     return [fileTempPath, filePath];
@@ -874,8 +949,8 @@ export const remove = async (filePath, linesToDelete) => {
         await exec(command);
         return [fileTempPath, filePath];
     }
-    catch {
-        return [fileTempPath, null];
+    catch (error) {
+        return failWrite(fileTempPath, error);
     }
 };
 /**
@@ -1094,7 +1169,7 @@ async function searchEqualsNative(filePath, patterns, field, searchIn, comparedA
  *
  * Note: Decodes each line for comparison and can handle complex queries with multiple conditions.
  */
-export const search = async (filePath, operator, comparedAtValue, logicalOperator, searchIn, field, limit, offset, readWholeFile) => {
+export const search = async (filePath, operator, comparedAtValue, logicalOperator, searchIn, field, limit, offset, readWholeFile, earlyStop) => {
     // Native fast path for exact-equality searches (whole-line `grep -x -F`).
     if (operator === "=" &&
         !Array.isArray(operator) &&
@@ -1116,6 +1191,7 @@ export const search = async (filePath, operator, comparedAtValue, logicalOperato
     let linesCount = 0;
     const linesNumbers = new Set();
     let fileHandle = null;
+    let destroyStreams = () => { };
     const config = field ?? {
         key: "BLABLA",
         type: "string",
@@ -1131,7 +1207,8 @@ export const search = async (filePath, operator, comparedAtValue, logicalOperato
         // Open the file for reading.
         fileHandle = await open(filePath, "r");
         // Create a Readline interface to read the file line by line.
-        const rl = createReadLineInternface(filePath, fileHandle);
+        const { rl, destroy } = openReadline(filePath, fileHandle);
+        destroyStreams = destroy;
         // Iterate through each line in the file.
         for await (const line of rl) {
             // Increment the line count for each line.
@@ -1162,6 +1239,10 @@ export const search = async (filePath, operator, comparedAtValue, logicalOperato
                 }
                 // Store the decoded line in the result object.
                 matchingLines[linesCount] = decodedLine;
+                // Every requested match is now in hand: stop instead of
+                // scanning the rest of the column (id -> line resolution).
+                if (earlyStop && limit && linesNumbers.size >= limit)
+                    break;
             }
         }
         // Convert the Map to an object using Object.fromEntries and return the result.
@@ -1173,7 +1254,9 @@ export const search = async (filePath, operator, comparedAtValue, logicalOperato
         return [null, 0, null];
     }
     finally {
-        // Close the file handle in the finally block to ensure it is closed even if an error occurs.
+        // Tear down the read streams (an early `break` must not leave a handle
+        // open) and then the file handle, even if an error occurred.
+        destroyStreams();
         await fileHandle?.close();
     }
 };

package/dist/index.d.ts

@@ -252,14 +252,6 @@ export default class Inibase {
     private sameCellValue;
     private isSimpleField;
     private processSimpleField;
-    /**
-     * Batched read for top-level simple columns: a single `paste | sed` child
-     * process returns every requested line of every column at once instead of
-     * spawning one `sed`/`gunzip|sed` child process per column. Returning null
-     * (e.g. when any column file is missing, or the shell command fails) makes
-     * the caller fall back to the existing per-column reads.
-     */
-    private processSimpleFieldsBatch;
     private isArrayField;
     private processArrayField;
     private isObjectField;

package/dist/index.js

@@ -110,6 +110,18 @@ function astHasHops(node) {
             return astHasHops(node.arg);
     }
 }
+/**
+ * Await a batch of column-write promises and surface the first rejection.
+ * `Promise.allSettled` never short-circuits, so a swallowed failure would let
+ * the caller publish a partial column set — the pagination row count and the
+ * column line counts would diverge. Every batch that fills a `renameList` must
+ * be inspected through here before the journal/pagination is touched.
+ */
+async function settleWrites(results) {
+    const failed = results.find((result) => result.status === "rejected");
+    if (failed)
+        throw failed.reason;
+}
 /**
  * @param {string} database - Database name
  * @param {string} [mainFolder="."] - Main folder path
@@ -982,19 +994,39 @@ export default class Inibase {
     }
     async processSchemaData(tableName, schema, linesNumber, options, prefix) {
         const RETURN = {};
-        // Fast path: read every top-level simple column in one `paste | sed`
-        // child process instead of spawning one child process per column.
-        // Nested (prefixed) schemas keep the per-column path.
+        // Sparse path: for an explicit finite selection, stream each top-level
+        // simple column only up to the last requested line (never to EOF) and
+        // decode it independently, instead of pasting every column through a
+        // shell pipeline that always scans the whole table and can misalign on
+        // values containing a literal tab. Nested (prefixed) schemas keep the
+        // per-column path below.
         let batchedKeys = null;
         if (!prefix && linesNumber?.length) {
             const simpleFields = schema.filter((field) => this.isSimpleField(field.type));
-            const batched = await this.processSimpleFieldsBatch(tableName, simpleFields, linesNumber);
-            if (batched) {
-                batchedKeys = new Set(simpleFields.map((field) => field.key));
-                for (const [line, row] of Object.entries(batched)) {
+            const decodeID = globalConfig[this.databasePath].tables?.get(tableName)?.config
+                .decodeID === true;
+            const extension = this.getFileExtension(tableName);
+            const reads = await Promise.all(simpleFields.map(async (field) => {
+                const fieldPath = join(this.databasePath, tableName, `${field.key}${extension}`);
+                if (!(await File.isExists(fieldPath)))
+                    return null;
+                const items = await File.getLines(fieldPath, linesNumber, {
+                    ...field,
+                    type: field.key === "id" && decodeID ? "number" : field.type,
+                    databasePath: this.databasePath,
+                });
+                return { key: field.key, items };
+            }));
+            batchedKeys = new Set(simpleFields.map((field) => field.key));
+            for (const read of reads) {
+                if (!read)
+                    continue;
+                for (const [line, value] of Object.entries(read.items)) {
+                    if (typeof value === "undefined")
+                        continue;
                     if (!RETURN[line])
                         RETURN[line] = {};
-                    Object.assign(RETURN[line], row);
+                    Object.assign(RETURN[line], { [read.key]: value });
                 }
             }
         }
@@ -1658,78 +1690,6 @@ export default class Inibase {
             }
         }
     }
-    /**
-     * Batched read for top-level simple columns: a single `paste | sed` child
-     * process returns every requested line of every column at once instead of
-     * spawning one `sed`/`gunzip|sed` child process per column. Returning null
-     * (e.g. when any column file is missing, or the shell command fails) makes
-     * the caller fall back to the existing per-column reads.
-     */
-    async processSimpleFieldsBatch(tableName, fields, linesNumber) {
-        if (!fields.length || !linesNumber.length)
-            return null;
-        const decodeID = globalConfig[this.databasePath].tables?.get(tableName)?.config
-            .decodeID === true;
-        // Decode configs are computed once per column (the per-column read path
-        // builds one config per file too), not once per cell.
-        const cols = [];
-        for (const field of fields) {
-            const path = join(this.databasePath, tableName, `${field.key}${this.getFileExtension(tableName)}`);
-            if (!(await File.isExists(path)))
-                return null;
-            cols.push({
-                path,
-                key: field.key,
-                config: {
-                    ...field,
-                    type: field.key === "id" && decodeID ? "number" : field.type,
-                    databasePath: this.databasePath,
-                },
-            });
-        }
-        const sortedLines = [...linesNumber].sort((a, b) => a - b);
-        const range = File._groupIntoRanges(sortedLines);
-        const files = cols.map(({ path }) => File.escapeShellPath(path)).join(" ");
-        const isGz = this.getFileExtension(tableName).endsWith(".gz");
-        // Each compressed column needs its own process substitution so the
-        // streams stay aligned column-by-column inside `paste`.
-        const pasteInputs = isGz
-            ? cols
-                .map(({ path }) => `<(gunzip -c ${File.escapeShellPath(path)})`)
-                .join(" ")
-            : files;
-        const command = isGz
-            ? `bash -c 'paste -d "\\t" ${pasteInputs} | sed -n "${range}"'`
-            : `paste -d'\\t' ${files} | sed -n '${range}'`;
-        let output;
-        try {
-            output = (await UtilsServer.exec(command)).stdout;
-        }
-        catch {
-            return null;
-        }
-        const outLines = output.trimEnd().split("\n");
-        const RETURN = {};
-        for (let i = 0; i < outLines.length && i < sortedLines.length; i++) {
-            const lineNo = sortedLines[i];
-            const cells = outLines[i].split("\t");
-            const row = {};
-            let added = false;
-            for (let c = 0; c < cols.length; c++) {
-                const raw = cells[c];
-                if (raw === undefined)
-                    continue;
-                const value = File.decode(raw, cols[c].config);
-                if (value !== undefined) {
-                    row[cols[c].key] = value;
-                    added = true;
-                }
-            }
-            if (added)
-                RETURN[lineNo] = row;
-        }
-        return Object.keys(RETURN).length ? RETURN : null;
-    }
     // Helper function to check if the field type is array
     isArrayField(fieldType) {
         return ((Array.isArray(fieldType) &&
@@ -2768,7 +2728,11 @@ export default class Inibase {
                 }
             }
             if (!lineNumbers) {
-                [lineNumbers, countItems] = await File.search(join(tablePath, `id${this.getFileExtension(tableName)}`), "[]", Ids.map((id) => Utils.isNumber(id) ? Number(id) : UtilsServer.decodeID(id)), undefined, undefined, { key: "BLABLA", type: "number" }, Ids.length, 0, !this.totalItems.has(`${tableName}-id`));
+                [lineNumbers, countItems] = await File.search(join(tablePath, `id${this.getFileExtension(tableName)}`), "[]", Ids.map((id) => Utils.isNumber(id) ? Number(id) : UtilsServer.decodeID(id)), undefined, undefined, { key: "BLABLA", type: "number" }, Ids.length, 0, 
+                // The caller only needs the matching line numbers, not a
+                // whole-table count: stop as soon as every requested id has
+                // been found instead of scanning the column to EOF.
+                false, true);
             }
             if (!lineNumbers)
                 return null;
@@ -2915,10 +2879,10 @@ export default class Inibase {
                     ? clonedData.toReversed()
                     : clonedData
                 : clonedData);
-            await Promise.allSettled(Object.entries(pathesContents).map(async ([path, content]) => renameList.push(globalConfig[this.databasePath].tables?.get(tableName)?.config
+            await settleWrites(await Promise.allSettled(Object.entries(pathesContents).map(async ([path, content]) => renameList.push(globalConfig[this.databasePath].tables?.get(tableName)?.config
                 .prepend
                 ? await File.prepend(path, content)
-                : await File.append(path, content))));
+                : await File.append(path, content)))));
             const newTotal = _totalItems + (Array.isArray(data) ? data.length : 1);
             const pagination = {
                 from: paginationFilePath,
@@ -3060,7 +3024,7 @@ export default class Inibase {
                     const evaluated = await this.evaluateComputedRows(tableName, computedPlan, mergedRows);
                     this.mergeComputedLineRecords(tableName, computedPlan, evaluated, pathesContents, existing);
                 }
-                await Promise.allSettled(Object.entries(pathesContents).map(async ([path, content]) => renameList.push(await File.replace(path, content, total))));
+                await settleWrites(await Promise.allSettled(Object.entries(pathesContents).map(async ([path, content]) => renameList.push(await File.replace(path, content, total)))));
                 if (this.activeTransaction) {
                     // Stage instead of publishing: row count is unchanged so
                     // there is no pagination rename to journal.
@@ -3157,7 +3121,7 @@ export default class Inibase {
                     }
                     this.mergeComputedLineRecords(tableName, computedPlan, evaluated, pathesContents, existing);
                 }
-                await Promise.allSettled(Object.entries(pathesContents).map(async ([path, content]) => renameList.push(await File.replace(path, content))));
+                await settleWrites(await Promise.allSettled(Object.entries(pathesContents).map(async ([path, content]) => renameList.push(await File.replace(path, content)))));
                 if (this.activeTransaction) {
                     await this.stageTxnOp(tableName, renameList, null);
                     txnStaged = true;
@@ -3330,7 +3294,7 @@ export default class Inibase {
                     const remaining = total - (Array.isArray(where) ? where.length : 1);
                     if (total && remaining > 0) {
                         this.idDensity.set(tableName, false);
-                        await Promise.allSettled(files.map(async (file) => renameList.push(await File.remove(join(tablePath, file), where))));
+                        await settleWrites(await Promise.allSettled(files.map(async (file) => renameList.push(await File.remove(join(tablePath, file), where)))));
                         const pagination = {
                             from: paginationFilePath,
                             to: join(tablePath, `${lastId}-${remaining}.pagination`),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "inibase",
-  "version": "3.6.0",
+  "version": "3.7.0",
   "type": "module",
   "author": {
     "name": "Karim Amahtil",