@push.rocks/smartdb 3.0.0 → 3.0.2

package/ts/ts_migration/migrators/v0_to_v1.ts

@@ -1,6 +1,6 @@
-import * as fs from 'fs';
-import * as path from 'path';
-import * as crypto from 'crypto';
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+import * as crypto from 'node:crypto';
 import { BSON } from 'bson';
 
 // ---------------------------------------------------------------------------
@@ -100,6 +100,141 @@ function crc32(data: Buffer): number {
   return (crc ^ 0xFFFFFFFF) >>> 0;
 }
 
+const pathExists = async (pathArg: string): Promise<boolean> => {
+  try {
+    await fs.stat(pathArg);
+    return true;
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false;
+    throw error;
+  }
+};
+
+export class SmartDbStorageMigrationCleanupError extends AggregateError {
+  constructor(operationErrorArg: unknown, cleanupErrorsArg: unknown[]) {
+    super(
+      [operationErrorArg, ...cleanupErrorsArg],
+      'SmartDB legacy storage migration failed and staging cleanup was incomplete',
+      { cause: operationErrorArg },
+    );
+    this.name = 'SmartDbStorageMigrationCleanupError';
+  }
+}
+
+export const runWithOwnedMigrationFileHandle = async <
+  TResult,
+  TFileHandle extends { close(): Promise<void> },
+>(
+  fileHandleArg: TFileHandle,
+  operationArg: (
+    fileHandleArg: TFileHandle,
+  ) => Promise<TResult>,
+  signalArg?: AbortSignal,
+): Promise<TResult> => {
+  let operationResult: TResult | undefined;
+  let operationError: unknown;
+  let operationCompleted = false;
+  try {
+    operationResult = await operationArg(fileHandleArg);
+    operationCompleted = true;
+  } catch (error) {
+    operationError = error;
+  }
+  try {
+    await fileHandleArg.close();
+  } catch (closeError) {
+    if (operationError !== undefined) {
+      throw new SmartDbStorageMigrationCleanupError(operationError, [closeError]);
+    }
+    if (signalArg?.aborted) {
+      throw new SmartDbStorageMigrationCleanupError(signalArg.reason, [closeError]);
+    }
+    throw closeError;
+  }
+  if (!operationCompleted) throw operationError;
+  return operationResult as TResult;
+};
+
+const withFileHandle = async <TResult>(
+  filePathArg: string,
+  flagsArg: 'r' | 'wx',
+  operationArg: (
+    fileHandleArg: Awaited<ReturnType<typeof fs.open>>,
+  ) => Promise<TResult>,
+  signalArg?: AbortSignal,
+): Promise<TResult> => runWithOwnedMigrationFileHandle(
+  await fs.open(filePathArg, flagsArg),
+  operationArg,
+  signalArg,
+);
+
+const writeBuffer = async (
+  fileHandleArg: Awaited<ReturnType<typeof fs.open>>,
+  bufferArg: Buffer,
+  signalArg?: AbortSignal,
+): Promise<void> => {
+  let offset = 0;
+  while (offset < bufferArg.length) {
+    signalArg?.throwIfAborted();
+    const { bytesWritten } = await fileHandleArg.write(
+      bufferArg,
+      offset,
+      bufferArg.length - offset,
+      null,
+    );
+    if (bytesWritten <= 0) {
+      throw new Error('SmartDB storage migration write made no progress');
+    }
+    offset += bytesWritten;
+  }
+  signalArg?.throwIfAborted();
+};
+
+const syncDirectory = async (
+  directoryPathArg: string,
+  signalArg?: AbortSignal,
+): Promise<void> => {
+  try {
+    await withFileHandle(directoryPathArg, 'r', async (directoryHandleArg) => {
+      await directoryHandleArg.sync();
+    }, signalArg);
+  } catch (error) {
+    if (!['EINVAL', 'ENOTSUP'].includes((error as NodeJS.ErrnoException).code || '')) {
+      throw error;
+    }
+  }
+};
+
+const syncFile = async (filePathArg: string, signalArg?: AbortSignal): Promise<void> => {
+  await withFileHandle(filePathArg, 'r', async (fileHandleArg) => {
+    await fileHandleArg.sync();
+  }, signalArg);
+};
+
+const assertPublishedMigrationComplete = async (
+  collectionDirectoryArg: string,
+): Promise<void> => {
+  for (const fileName of ['data.rdb', 'keydir.hint', 'indexes.json']) {
+    let fileStat: Awaited<ReturnType<typeof fs.lstat>>;
+    try {
+      fileStat = await fs.lstat(path.join(collectionDirectoryArg, fileName));
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
+        throw new Error(
+          `SmartDB legacy migration target is incomplete: ${collectionDirectoryArg}`,
+          { cause: error },
+        );
+      }
+      throw error;
+    }
+    if (!fileStat.isFile() || fileStat.isSymbolicLink()) {
+      throw new Error(
+        `SmartDB legacy migration target is incomplete: ${collectionDirectoryArg}`,
+      );
+    }
+  }
+};
+
 // ---------------------------------------------------------------------------
 // Migration: v0 (JSON) → v1 (Bitcask binary)
 // ---------------------------------------------------------------------------
@@ -119,21 +254,27 @@ interface IKeyDirEntry {
  * - Returns a list of old files that can be safely deleted.
  * - On failure, cleans up any partial new files and throws.
  */
-export async function migrateV0ToV1(storagePath: string): Promise<string[]> {
+export async function migrateV0ToV1(
+  storagePath: string,
+  signalArg?: AbortSignal,
+): Promise<string[]> {
   const deletableFiles: string[] = [];
-  const createdDirs: string[] = [];
+  const stagingDirectories = new Set<string>();
 
   try {
-    const dbEntries = fs.readdirSync(storagePath, { withFileTypes: true });
+    signalArg?.throwIfAborted();
+    const dbEntries = await fs.readdir(storagePath, { withFileTypes: true });
 
     for (const dbEntry of dbEntries) {
+      signalArg?.throwIfAborted();
       if (!dbEntry.isDirectory()) continue;
       if (dbEntry.name.startsWith('.__rustdb_')) continue;
 
       const dbDir = path.join(storagePath, dbEntry.name);
-      const collFiles = fs.readdirSync(dbDir, { withFileTypes: true });
+      const collFiles = await fs.readdir(dbDir, { withFileTypes: true });
 
       for (const collFile of collFiles) {
+        signalArg?.throwIfAborted();
         if (!collFile.isFile()) continue;
         if (collFile.name.startsWith('.__rustdb_')) continue;
         if (!collFile.name.endsWith('.json')) continue;
@@ -145,35 +286,46 @@ export async function migrateV0ToV1(storagePath: string): Promise<string[]> {
 
         // Target directory
         const collDir = path.join(dbDir, collName);
-        if (fs.existsSync(collDir)) {
-          // Already migrated
+        if (await pathExists(collDir)) {
+          await assertPublishedMigrationComplete(collDir);
+          if (await pathExists(indexJsonPath)) deletableFiles.push(indexJsonPath);
+          deletableFiles.push(jsonPath);
           continue;
         }
 
         console.log(`[smartdb]   Migrating ${dbEntry.name}.${collName}...`);
 
         // Read the JSON collection
-        const jsonData = fs.readFileSync(jsonPath, 'utf-8');
+        const jsonData = await fs.readFile(jsonPath, {
+          encoding: 'utf8',
+          signal: signalArg,
+        });
+        signalArg?.throwIfAborted();
         const docs: any[] = JSON.parse(jsonData);
 
-        // Create collection directory
-        fs.mkdirSync(collDir, { recursive: true });
-        createdDirs.push(collDir);
+        // Build the complete collection in a hidden sibling and publish it
+        // atomically so cancellation can never expose partial v1 state.
+        const stagingDir = path.join(
+          dbDir,
+          `.__rustdb_migration_${collName}_${process.pid}_${crypto.randomBytes(8).toString('hex')}`,
+        );
+        await fs.mkdir(stagingDir);
+        stagingDirectories.add(stagingDir);
+        signalArg?.throwIfAborted();
 
         // Write data.rdb
-        const dataPath = path.join(collDir, 'data.rdb');
-        const fd = fs.openSync(dataPath, 'w');
-
-        try {
+        const dataPath = path.join(stagingDir, 'data.rdb');
+        await withFileHandle(dataPath, 'wx', async (dataFileHandle) => {
           // File header
           const headerBuf = writeFileHeader(FILE_TYPE_DATA);
-          fs.writeSync(fd, headerBuf);
+          await writeBuffer(dataFileHandle, headerBuf, signalArg);
 
           let currentOffset = BigInt(FILE_HEADER_SIZE);
           const keydir: Map<string, IKeyDirEntry> = new Map();
           const ts = BigInt(Date.now());
 
           for (const doc of docs) {
+            signalArg?.throwIfAborted();
             // Extract _id
             let idHex: string;
             if (doc._id && doc._id.$oid) {
@@ -194,7 +346,7 @@ export async function migrateV0ToV1(storagePath: string): Promise<string[]> {
             const valueBuf = Buffer.from(bsonBytes);
 
             const record = encodeDataRecord(ts, keyBuf, valueBuf);
-            fs.writeSync(fd, record);
+            await writeBuffer(dataFileHandle, record, signalArg);
 
             keydir.set(idHex, {
               offset: currentOffset,
@@ -206,47 +358,75 @@ export async function migrateV0ToV1(storagePath: string): Promise<string[]> {
             currentOffset += BigInt(record.length);
           }
 
-          fs.fsyncSync(fd);
-          fs.closeSync(fd);
+          await dataFileHandle.sync();
+          signalArg?.throwIfAborted();
 
           // Write keydir.hint
-          const hintPath = path.join(collDir, 'keydir.hint');
-          const hintFd = fs.openSync(hintPath, 'w');
-          fs.writeSync(hintFd, writeFileHeader(FILE_TYPE_HINT));
-          for (const [key, entry] of keydir) {
-            fs.writeSync(hintFd, encodeHintEntry(key, entry.offset, entry.recordLen, entry.valueLen, entry.timestamp));
-          }
-          fs.fsyncSync(hintFd);
-          fs.closeSync(hintFd);
-
-        } catch (writeErr) {
-          // Clean up on write failure
-          try { fs.closeSync(fd); } catch {}
-          throw writeErr;
-        }
+          const hintPath = path.join(stagingDir, 'keydir.hint');
+          await withFileHandle(hintPath, 'wx', async (hintFileHandle) => {
+            await writeBuffer(hintFileHandle, writeFileHeader(FILE_TYPE_HINT), signalArg);
+            for (const [key, entry] of keydir) {
+              signalArg?.throwIfAborted();
+              await writeBuffer(
+                hintFileHandle,
+                encodeHintEntry(
+                  key,
+                  entry.offset,
+                  entry.recordLen,
+                  entry.valueLen,
+                  entry.timestamp,
+                ),
+                signalArg,
+              );
+            }
+            await hintFileHandle.sync();
+            signalArg?.throwIfAborted();
+          }, signalArg);
+        }, signalArg);
 
         // Copy indexes.json if it exists
-        if (fs.existsSync(indexJsonPath)) {
-          const destIndexPath = path.join(collDir, 'indexes.json');
-          fs.copyFileSync(indexJsonPath, destIndexPath);
+        if (await pathExists(indexJsonPath)) {
+          const destIndexPath = path.join(stagingDir, 'indexes.json');
+          await fs.copyFile(indexJsonPath, destIndexPath);
+          await syncFile(destIndexPath, signalArg);
+          signalArg?.throwIfAborted();
           deletableFiles.push(indexJsonPath);
         } else {
           // Write default _id index
-          const destIndexPath = path.join(collDir, 'indexes.json');
-          fs.writeFileSync(destIndexPath, JSON.stringify([{ name: '_id_', key: { _id: 1 } }], null, 2));
+          const destIndexPath = path.join(stagingDir, 'indexes.json');
+          await fs.writeFile(
+            destIndexPath,
+            JSON.stringify([{ name: '_id_', key: { _id: 1 } }], null, 2),
+            { signal: signalArg },
+          );
+          await syncFile(destIndexPath, signalArg);
+          signalArg?.throwIfAborted();
         }
 
+        await assertPublishedMigrationComplete(stagingDir);
+        await syncDirectory(stagingDir, signalArg);
+        signalArg?.throwIfAborted();
+        await fs.rename(stagingDir, collDir);
+        stagingDirectories.delete(stagingDir);
+        await syncDirectory(dbDir, signalArg);
+        signalArg?.throwIfAborted();
         deletableFiles.push(jsonPath);
 
         console.log(`[smartdb]   Migrated ${dbEntry.name}.${collName}: ${docs.length} documents`);
       }
     }
   } catch (err) {
-    // Clean up any partially created directories
-    for (const dir of createdDirs) {
+    const cleanupErrors: unknown[] = [];
+    for (const dir of Array.from(stagingDirectories).reverse()) {
       try {
-        fs.rmSync(dir, { recursive: true, force: true });
-      } catch {}
+        await fs.rm(dir, { recursive: true, force: true });
+        await syncDirectory(path.dirname(dir));
+      } catch (cleanupError) {
+        cleanupErrors.push(cleanupError);
+      }
+    }
+    if (cleanupErrors.length > 0) {
+      throw new SmartDbStorageMigrationCleanupError(err, cleanupErrors);
     }
     throw err;
   }

package/ts/ts_migration/migrators/v1_auth_metadata_permissions.ts

@@ -1,4 +1,5 @@
 import type { TAuthMetadataPermissionTransitionStatus } from '../../ts_smartdb/rust-db-bridge.js';
+import type { ISmartDbManagementOperationOptions } from '../../ts_smartdb/service-types.js';
 
 const legacyAuthMetadataMode = 0o644;
 const privateAuthMetadataMode = 0o600;
@@ -8,6 +9,7 @@ export interface IAuthMetadataPermissionMigrationBridge {
     usersPathArg: string,
     expectedModeArg: number,
     targetModeArg: number,
+    optionsArg?: ISmartDbManagementOperationOptions,
   ): Promise<TAuthMetadataPermissionTransitionStatus>;
 }
 
@@ -22,9 +24,11 @@ export type TAuthMetadataPermissionMigrationStatus =
 export const migrateV1AuthMetadataPermissions = async (
   bridgeArg: IAuthMetadataPermissionMigrationBridge,
   usersPathArg: string,
+  optionsArg?: ISmartDbManagementOperationOptions,
 ): Promise<TAuthMetadataPermissionMigrationStatus> => bridgeArg
   .transitionAuthMetadataPermissions(
     usersPathArg,
     legacyAuthMetadataMode,
     privateAuthMetadataMode,
+    optionsArg,
   );

package/ts/ts_smartdb/rust-db-bridge.ts

@@ -537,20 +537,41 @@ export class RustDbBridge extends EventEmitter {
     usersPathArg: string,
     expectedModeArg: number,
     targetModeArg: number,
+    optionsArg?: ISmartDbManagementOperationOptions,
   ): Promise<TAuthMetadataPermissionTransitionStatus> {
-    const result = await this.bridge.sendCommand(
-      'transitionAuthMetadataPermissions',
-      {
-        usersPath: usersPathArg,
-        expectedMode: expectedModeArg,
-        targetMode: targetModeArg,
-      },
-    ) as { status: TAuthMetadataPermissionTransitionStatus };
-    return result.status;
+    validateSmartDbManagementOperationOptions(optionsArg);
+    try {
+      const result = await this.bridge.sendCommand(
+        'transitionAuthMetadataPermissions',
+        {
+          usersPath: usersPathArg,
+          expectedMode: expectedModeArg,
+          targetMode: targetModeArg,
+        },
+        optionsArg,
+      ) as { status: TAuthMetadataPermissionTransitionStatus };
+      return result.status;
+    } catch (error) {
+      await this.awaitManagementOperationTermination(error);
+      throw error;
+    }
   }
 
-  public async startDb(config: ISmartDbRustConfig): Promise<{ connectionUri: string }> {
-    return await this.bridge.sendCommand('start', { config }) as { connectionUri: string };
+  public async startDb(
+    config: ISmartDbRustConfig,
+    optionsArg?: ISmartDbManagementOperationOptions,
+  ): Promise<{ connectionUri: string }> {
+    validateSmartDbManagementOperationOptions(optionsArg);
+    try {
+      return await this.bridge.sendCommand(
+        'start',
+        { config },
+        optionsArg,
+      ) as { connectionUri: string };
+    } catch (error) {
+      await this.awaitManagementOperationTermination(error);
+      throw error;
+    }
   }
 
   public async stopDb(): Promise<void> {

package/ts/ts_smartdb/server/SmartdbServer.ts

@@ -2,7 +2,12 @@ import {
   RustDbBridge,
   SmartDbManagementOperationTerminationError,
 } from '../rust-db-bridge.js';
-import { AuthMetadataMigrationRunner, StorageMigrator } from '../../ts_migration/index.js';
+import {
+  AuthMetadataMigrationRunner,
+  SmartDbStorageMigrationCleanupError,
+  StorageMigrator,
+} from '../../ts_migration/index.js';
+import { validateSmartDbManagementOperationOptions } from '../offline-inspection.js';
 import type {
   IOpLogEntry,
   IOpLogResult,
@@ -148,11 +153,63 @@ export class SmartdbServer {
   /**
    * Start the server
    */
-  async start(): Promise<void> {
+  async start(optionsArg?: ISmartDbManagementOperationOptions): Promise<void> {
+    validateSmartDbManagementOperationOptions(optionsArg);
+    optionsArg?.signal?.throwIfAborted();
     if (this.isRunning || this.startInProgress || this.bridgeOwned) {
       throw new Error('Server is already running or still owns a Rust bridge');
     }
 
+    const startupStartedAt = Date.now();
+    const startupDeadline = optionsArg?.timeoutMs === undefined
+      ? undefined
+      : startupStartedAt + optionsArg.timeoutMs;
+    const timeoutError = new DOMException(
+      optionsArg?.timeoutMs === undefined
+        ? 'SmartDB startup timed out'
+        : `SmartDB startup timed out after ${optionsArg.timeoutMs}ms`,
+      'TimeoutError',
+    );
+    const startupAbortController = new AbortController();
+    let startupTimeout: ReturnType<typeof setTimeout> | undefined;
+    let externalAbortHandler: (() => void) | undefined;
+    if (optionsArg?.signal) {
+      externalAbortHandler = () => {
+        startupAbortController.abort(optionsArg.signal!.reason);
+      };
+      optionsArg.signal.addEventListener('abort', externalAbortHandler, { once: true });
+    }
+    if (optionsArg?.timeoutMs !== undefined) {
+      startupTimeout = setTimeout(() => {
+        startupAbortController.abort(timeoutError);
+      }, optionsArg.timeoutMs);
+    }
+    const checkpoint = () => {
+      if (
+        startupDeadline !== undefined
+        && Date.now() >= startupDeadline
+        && !startupAbortController.signal.aborted
+      ) {
+        startupAbortController.abort(timeoutError);
+      }
+      startupAbortController.signal.throwIfAborted();
+    };
+    const getBridgeOperationOptions = (): ISmartDbManagementOperationOptions | undefined => {
+      checkpoint();
+      const remainingTimeoutMs = startupDeadline === undefined
+        ? undefined
+        : startupDeadline - Date.now();
+      if (remainingTimeoutMs !== undefined && remainingTimeoutMs <= 0) {
+        startupAbortController.abort(timeoutError);
+        checkpoint();
+      }
+      if (!optionsArg?.signal && remainingTimeoutMs === undefined) return undefined;
+      return {
+        ...(optionsArg?.signal ? { signal: optionsArg.signal } : {}),
+        ...(remainingTimeoutMs === undefined ? {} : { timeoutMs: remainingTimeoutMs }),
+      };
+    };
+
     this.startInProgress = true;
     let resolveStartCompletion!: () => void;
     const startCompletionPromise = new Promise<void>((resolveArg) => {
@@ -160,16 +217,19 @@ export class SmartdbServer {
     });
     this.startCompletionPromise = startCompletionPromise;
     try {
+      checkpoint();
       // Run storage migration for file-based storage before starting Rust engine
       if (this.options.storage === 'file' && this.options.storagePath) {
         const migrator = new StorageMigrator(this.options.storagePath);
-        await migrator.run();
+        await migrator.run(startupAbortController.signal);
+        checkpoint();
       }
 
       // Ownership starts before spawn because a rejected spawn can still have
       // created a child whose termination must be confirmed.
       this.bridgeOwned = true;
-      const spawned = await this.bridge.spawn();
+      const spawned = await this.bridge.spawn(getBridgeOperationOptions());
+      checkpoint();
       if (!spawned) {
         throw new Error(
           'smartdb Rust binary not found. Set SMARTDB_RUST_BINARY env var, ' +
@@ -181,7 +241,11 @@ export class SmartdbServer {
         const authMetadataMigrationRunner = new AuthMetadataMigrationRunner(
           this.options.auth.usersPath,
         );
-        await authMetadataMigrationRunner.run(this.bridge);
+        await authMetadataMigrationRunner.run(
+          this.bridge,
+          getBridgeOperationOptions(),
+        );
+        checkpoint();
       }
 
       // Send config, get back connectionUri
@@ -196,7 +260,8 @@ export class SmartdbServer {
         auth: this.options.auth,
         tls: this.options.tls,
         oplog: this.options.oplog,
-      });
+      }, getBridgeOperationOptions());
+      checkpoint();
       if (!this.bridgeOwned || !this.bridge.running) {
         throw new Error('smartdb Rust process exited during startup');
       }
@@ -216,8 +281,19 @@ export class SmartdbServer {
         this.resolvedPort = resolvedPort;
       }
       this.resolvedConnectionUri = result.connectionUri;
+      checkpoint();
       this.isRunning = true;
     } catch (error) {
+      const startupError = error instanceof SmartDbManagementOperationTerminationError
+        || error instanceof SmartDbStorageMigrationCleanupError
+        ? error
+        : optionsArg?.signal?.aborted
+          ? optionsArg.signal.reason
+          : startupAbortController.signal.aborted || (
+              startupDeadline !== undefined && Date.now() >= startupDeadline
+            )
+            ? timeoutError
+            : error;
       this.isRunning = false;
       this.resolvedConnectionUri = '';
       this.resolvedPort = undefined;
@@ -227,14 +303,18 @@ export class SmartdbServer {
           this.bridgeOwned = false;
         } catch (cleanupError) {
           throw new AggregateError(
-            [error, cleanupError],
+            [startupError, cleanupError],
             'SmartDB startup failed and Rust bridge cleanup was incomplete',
-            { cause: error },
+            { cause: startupError },
           );
         }
       }
-      throw error;
+      throw startupError;
     } finally {
+      if (startupTimeout) clearTimeout(startupTimeout);
+      if (externalAbortHandler && optionsArg?.signal) {
+        optionsArg.signal.removeEventListener('abort', externalAbortHandler);
+      }
       this.startInProgress = false;
       resolveStartCompletion();
       if (this.startCompletionPromise === startCompletionPromise) {