inibase 3.6.0 → 3.7.1

package/dist/index.d.ts

@@ -44,6 +44,14 @@ export interface TableConfig {
     cache?: boolean;
     prepend?: boolean;
     decodeID?: boolean;
+    /**
+     * Opt a table into the incremental update log (format 2): `put` appends a
+     * small frame instead of rewriting whole columns, and reads merge the log
+     * over the base columns. Requires a schema of simple top-level fields and
+     * is incompatible with `prepend`, transactions and DDL on the table
+     * (those need compaction, shipped in 3.9).
+     */
+    log?: boolean;
 }
 export interface TableObject {
     schema?: Schema;
@@ -88,7 +96,7 @@ declare global {
         entries<T extends object>(o: T): Entries<T>;
     }
 }
-export declare const ERROR_CODES: readonly ["GROUP_UNIQUE", "FIELD_UNIQUE", "FIELD_REQUIRED", "NO_SCHEMA", "TABLE_EMPTY", "INVALID_ID", "INVALID_TYPE", "INVALID_PARAMETERS", "NO_ENV", "TABLE_EXISTS", "TABLE_NOT_EXISTS", "INVALID_REGEX_MATCH", "INVALID_NAME", "COMPUTED_FIELD_SYNTAX", "COMPUTED_FIELD_UNKNOWN_FIELD", "COMPUTED_FIELD_INVALID_LINK", "COMPUTED_FIELD_INVALID_TARGET", "COMPUTED_FIELD_CONFLICT", "COMPUTED_FIELD_CYCLE", "COMPUTED_FIELD_SETTABLE", "COMPUTED_FIELD_DANGLING_LINK", "COMPUTED_FIELD_ARITHMETIC"];
+export declare const ERROR_CODES: readonly ["GROUP_UNIQUE", "FIELD_UNIQUE", "FIELD_REQUIRED", "NO_SCHEMA", "TABLE_EMPTY", "INVALID_ID", "INVALID_TYPE", "INVALID_PARAMETERS", "NO_ENV", "TABLE_EXISTS", "TABLE_NOT_EXISTS", "INVALID_REGEX_MATCH", "INVALID_NAME", "COMPUTED_FIELD_SYNTAX", "COMPUTED_FIELD_UNKNOWN_FIELD", "COMPUTED_FIELD_INVALID_LINK", "COMPUTED_FIELD_INVALID_TARGET", "COMPUTED_FIELD_CONFLICT", "COMPUTED_FIELD_CYCLE", "COMPUTED_FIELD_SETTABLE", "COMPUTED_FIELD_DANGLING_LINK", "COMPUTED_FIELD_ARITHMETIC", "UPDATE_LOG_CORRUPT", "UPDATE_LOG_FRAME_TOO_LARGE", "MAINTENANCE_REQUIRED", "UNSUPPORTED_FORMAT"];
 export type ErrorCode = (typeof ERROR_CODES)[number];
 export type ErrorLang = "en" | "ar" | "fr" | "es";
 export declare const globalConfig: {
@@ -121,6 +129,15 @@ export default class Inibase {
      *  changes: see the invalidation in `getTable`'s reload branch,
      *  `createTable` and `updateTableLocked`. Bounded by the number of tables. */
     private readonly computedPlanCache;
+    /** Per-table `.format` marker + live update-log overlays (format 2 tables).
+     *  Keyed by `databasePath\0tableName` so two databases in one process never
+     *  share state. Populated lazily by `getOverlay`. */
+    private readonly tableFormats;
+    private readonly overlays;
+    /** Last observed identity of each table's `.format` file (`mtime:size:ino`),
+     *  used to detect a compaction by another process without re-reading the
+     *  marker on every access. */
+    private readonly formatStamps;
     private databasePath;
     private uniqueMap;
     private schemaFileExtension;
@@ -176,6 +193,74 @@ export default class Inibase {
      * @return {*}  {Promise<TableObject | undefined>}
      */
     getTable(tableName: string): Promise<TableObject | undefined>;
+    private overlayKey;
+    /** True when the table opted into the incremental update log. */
+    private isLogTable;
+    /**
+     * Resolve a table's format marker, revalidating a cached *log* table when
+     * its `.format` file changed underneath us. A compaction by another process
+     * bumps `baseGen` and empties the log; without re-reading the marker this
+     * process would scan the new (empty) log with the old `baseGen`, and once a
+     * post-compaction frame appears the generation gap would look like
+     * corruption. Legacy tables never change format implicitly, so they skip
+     * the stat entirely and keep the pre-3.8 read path cost.
+     */
+    /**
+     * Read a table's `.format` marker and refuse a table written by a newer
+     * engine (§4.1) *before* any column is read: a higher `format`, or a
+     * `minReader` above this build, throws UNSUPPORTED_FORMAT rather than being
+     * silently mis-read as format 2/legacy.
+     */
+    private readValidatedFormat;
+    private currentFormat;
+    /**
+     * Load (or refresh) a table's live overlay. Returns null for legacy tables
+     * (no `.format` marker) so every existing code path stays untouched. With
+     * `forWrite`, repairs a torn tail and rebuilds to the committed boundary;
+     * the caller must already hold the table writer lock.
+     */
+    private getOverlay;
+    private stampFormat;
+    /**
+     * The format-2 update log stores one encoded cell per changed *top-level
+     * simple* column; nested (object/array/table) and computed fields are not
+     * representable without a compaction step (3.9). A log table whose schema
+     * violates this is rejected at DDL time so the write path can always take
+     * the log and never needs to interleave a whole-column rewrite.
+     */
+    private assertLogCompatibleSchema;
+    private idFieldOf;
+    /** Read the internal numeric id for each requested line (stable overlay key). */
+    private overlayLineIds;
+    /** Internal id -> current line number(s), built from the id column. */
+    private overlayIdToLines;
+    /**
+     * Per-line raw-cell overrides for one column, from the overlay. Used by
+     * `File.search` so criteria, uniqueness and aggregate scans see the merged
+     * (un-compacted) values. Returns undefined when the table has no overlay or
+     * the column is never overlaid.
+     */
+    private overlayColumnOverrides;
+    /**
+     * Fold every logged update into the base columns and reset the log.
+     *
+     * Safe to call on any table: legacy tables (no `.format` marker) are a
+     * no-op. Semantically neutral — it only materializes already-acknowledged
+     * updates, so a failure never makes an acknowledged `put` disappear.
+     * Returns the new `baseGen` for a compacted table, or `null` when there was
+     * nothing to fold.
+     */
+    compact(tableName: string): Promise<number | null>;
+    /**
+     * Compaction core. The caller must hold the table writer lock. Streams each
+     * overlaid base column through `File.replace` (which handles compression),
+     * then publishes the new columns, an empty `updates.log`, and a `.format`
+     * marker with the bumped generation in one journaled commit, so a crash
+     * either rolls every file back (old base + old log + old `baseGen`) or
+     * forward (new base + empty log + new `baseGen`). No acknowledged update is
+     * ever lost.
+     */
+    private compactLogTable;
     getTableSchema(tableName: string): Promise<Schema>;
     private throwErrorIfTableEmpty;
     validateData(data: Data | Data[], schema: Schema, skipRequiredField?: boolean): void;
@@ -190,6 +275,17 @@ export default class Inibase {
     private joinPathesContents;
     private _processSchemaDataHelper;
     private processSchemaData;
+    /**
+     * Fold the table's un-compacted update-log overlay onto decoded rows.
+     *
+     * `RETURN` is keyed by line number and its `id` is the user-facing id, so
+     * the overlay (keyed by the internal numeric id) is matched through a fresh
+     * id-column read of just the requested lines. Only columns present in
+     * `schema` (which is already narrowed to the requested `columns`) are
+     * written, so a partial projection never grows extra keys. Legacy tables
+     * (no `.format` marker) return immediately.
+     */
+    private mergeOverlayIntoRows;
     /**
      * Extract the raw expression from a field's `computed` property (string or
      * persisted `{ expr, ast }` spec).
@@ -252,14 +348,6 @@ export default class Inibase {
     private sameCellValue;
     private isSimpleField;
     private processSimpleField;
-    /**
-     * Batched read for top-level simple columns: a single `paste | sed` child
-     * process returns every requested line of every column at once instead of
-     * spawning one `sed`/`gunzip|sed` child process per column. Returning null
-     * (e.g. when any column file is missing, or the shell command fails) makes
-     * the caller fall back to the existing per-column reads.
-     */
-    private processSimpleFieldsBatch;
     private isArrayField;
     private processArrayField;
     private isObjectField;
@@ -390,6 +478,13 @@ export default class Inibase {
     get<_TData extends Record<string, any> & Partial<Data>>(tableName: string, where: string | number | (string | number)[] | Criteria | undefined, options: Options | undefined, onlyOne: true, onlyLinesNumbers: true, _whereIsLinesNumbers?: boolean): Promise<number | null>;
     private _get;
     private getOnce;
+    /**
+     * In-process sort fallback for log tables with a non-empty overlay: the
+     * shell `paste | sort | awk` path reads base cells and would miss puts.
+     * Reads the merged rows for the (optionally filtered) lines, sorts them by
+     * the requested keys with type-aware comparison, then paginates.
+     */
+    private getOnceSortedWithOverlay;
     /**
      * Create new item(s) in a table
      *
@@ -422,6 +517,16 @@ export default class Inibase {
     put<TData extends Record<string, any> & Partial<Data>>(tableName: string, data: Data & TData, where: number | string | (number | string)[] | Criteria | undefined, options: Options | undefined, returnUpdatedData: true | boolean, _whereIsLinesNumbers?: boolean): Promise<(Data & TData) | null>;
     put<TData extends Record<string, any> & Partial<Data>>(tableName: string, data: (Data & TData)[], where: number | string | (number | string)[] | Criteria | undefined, options: Options | undefined, returnUpdatedData: true | boolean, _whereIsLinesNumbers?: boolean): Promise<(Data & TData)[] | null>;
     put<TData extends Record<string, any> & Partial<Data>>(tableName: string, data: (Data & TData) | (Data & TData)[], where: number | string | (number | string)[] | Criteria | undefined, options: Options | undefined, returnUpdatedData: true | boolean, _whereIsLinesNumbers?: boolean): Promise<(Data & TData) | (Data & TData)[] | null>;
+    /**
+     * Incremental `put` for format-2 (log) tables.
+     *
+     * Resolves the target rows to their stable internal ids, validates the
+     * payload against the merged view (uniqueness included), and appends one
+     * bounded frame instead of rewriting whole columns. The base files are left
+     * untouched; readers merge the overlay. Returns `undefined` when nothing
+     * matches, matching the legacy `put` contract.
+     */
+    private putViaLog;
     private _put;
     /**
      * Delete item(s) in a table