agl 21.0.2 → 22.0.0

package/ts_migration/v23_runtimeconfig.ts

@@ -0,0 +1,112 @@
+import * as plugins from '../ts/plugins.js';
+import {
+  assertControllerAuthDocument,
+  ControllerAuthModel,
+} from '../ts/classes.authmodels.js';
+import { AuthError, type IControllerAuthDocument } from '../ts/interfaces.auth.js';
+
+const migrationPageLimit = 128;
+const assertRuntimeConfigDocument: (
+  valueArg: unknown,
+) => asserts valueArg is IControllerAuthDocument = assertControllerAuthDocument;
+
+const persistedBodyFromRawDocument = (
+  valueArg: Record<string, unknown>,
+): Record<string, unknown> => {
+  const body = { ...valueArg };
+  delete body._id;
+  delete body._smartdataRevision;
+  return body;
+};
+
+const isAbsoluteNormalizedPath = (valueArg: unknown): valueArg is string => (
+  typeof valueArg === 'string'
+  && valueArg.length > 0
+  && plugins.path.isAbsolute(valueArg)
+  && plugins.path.normalize(valueArg) === valueArg
+);
+
+export const migrateV23RuntimeConfigDocument = (
+  valueArg: Record<string, unknown>,
+): { document: IControllerAuthDocument; migrated: boolean } => {
+  const config = valueArg.config;
+  if (!config || typeof config !== 'object' || Array.isArray(config)) {
+    throw new Error('Invalid legacy controller runtime configuration.');
+  }
+  const configRecord = config as Record<string, unknown>;
+  if (!Object.hasOwn(configRecord, 'workspaceDirectory')) {
+    assertRuntimeConfigDocument(valueArg);
+    return { document: valueArg, migrated: false };
+  }
+  if (!isAbsoluteNormalizedPath(configRecord.workspaceDirectory)) {
+    throw new Error('Legacy controller workspace is not an absolute normalized path.');
+  }
+  const projectsRoot = Object.hasOwn(configRecord, 'projectsRoot')
+    ? configRecord.projectsRoot
+    : configRecord.workspaceDirectory;
+  if (!isAbsoluteNormalizedPath(projectsRoot)) {
+    throw new Error('Legacy controller projects root is not an absolute normalized path.');
+  }
+  const migratedConfig: Record<string, unknown> = { ...configRecord, projectsRoot };
+  delete migratedConfig.workspaceDirectory;
+  const migrated: Record<string, unknown> = { ...valueArg, config: migratedConfig };
+  assertRuntimeConfigDocument(migrated);
+  return { document: migrated, migrated: true };
+};
+
+export class RuntimeConfigV23Migration {
+  constructor(private readonly database: plugins.smartdata.SmartdataDb | undefined) {}
+
+  public async run(): Promise<void> {
+    if (!this.database) {
+      throw new AuthError('not_initialized', 'The authentication store database is unavailable.');
+    }
+    const collection = ControllerAuthModel.collection.mongoDbCollection;
+    let lastId: plugins.smartdata.TStoredDocument<IControllerAuthDocument>['_id'] | undefined;
+    while (true) {
+      const cursor = collection.find(
+        lastId ? { _id: { $gt: lastId } } : {},
+      ).sort({ _id: 1 }).limit(migrationPageLimit);
+      let documents: Awaited<ReturnType<typeof cursor.toArray>>;
+      try {
+        documents = await cursor.toArray();
+      } finally {
+        await cursor.close();
+      }
+      for (const raw of documents) {
+        const migration = migrateV23RuntimeConfigDocument(
+          persistedBodyFromRawDocument(raw),
+        );
+        if (!migration.migrated) continue;
+        const selector = raw._smartdataRevision === undefined
+          ? { _id: raw._id, _smartdataRevision: { $exists: false } }
+          : { _id: raw._id, _smartdataRevision: raw._smartdataRevision };
+        const replaced = await collection.findOneAndReplace(
+          selector,
+          { _id: raw._id, ...migration.document, _smartdataRevision: plugins.crypto.randomUUID() },
+          { returnDocument: 'after', includeResultMetadata: false, upsert: false },
+        );
+        if (replaced) continue;
+        const concurrent = await collection.findOne({ _id: raw._id });
+        if (!concurrent) {
+          throw new AuthError(
+            'concurrent_change',
+            'The runtime configuration migration changed concurrently.',
+          );
+        }
+        const reconciled = migrateV23RuntimeConfigDocument(
+          persistedBodyFromRawDocument(concurrent),
+        );
+        if (reconciled.migrated) {
+          throw new AuthError(
+            'concurrent_change',
+            'The runtime configuration migration changed concurrently.',
+          );
+        }
+      }
+      const last = documents.at(-1);
+      if (!last || documents.length < migrationPageLimit) break;
+      lastId = last._id;
+    }
+  }
+}

package/ts_migration/v2_controllerdataroot.ts

@@ -31,6 +31,7 @@ const maximumUpgradeLogs = 64;
 const maximumLockArtifacts = 64;
 const maximumLockAttempts = 200;
 const lockWaitMilliseconds = 50;
+const lockGuardInitializationGraceMs = 30_000;
 
 const credentialHashPattern = /^[a-f0-9]{64}$/;
 const digestPattern = /^[a-f0-9]{64}$/;
@@ -39,6 +40,7 @@ const decimalPattern = /^(0|[1-9][0-9]*)$/;
 const controllerLogPattern = /^controller-([1-9][0-9]{0,4})\.log(?:\.old)?$/;
 const upgradeLogPattern = /^upgrade-[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{6}\.[0-9]{3}Z-[A-Za-z0-9_-]{10}\.log$/;
 const embeddedDatabaseJournalTemporaryPattern = /^\.smartdb-location-migration\.json\.[1-9][0-9]*-[a-f0-9]{16}\.tmp$/;
+const lockTemporarySuffixPattern = /^\.tmp-([0-9]+)-([1-9][0-9]*)-([a-f0-9]{64})$/;
 
 export type TControllerDataRootMigrationPhase =
   | 'legacy-authoritative'
@@ -116,6 +118,7 @@ export interface IControllerDataRootMigrationOptions {
   newEmbeddedSocketPath: string;
   invokerPid: number;
   listDataWriterProcesses: () => Promise<readonly IControllerDataWriterProcessRecord[]>;
+  preserveEmbeddedDataDirectory?: boolean;
   isSocketListening?: (socketPathArg: string) => Promise<boolean>;
   relocateCredentialStore?: (
     intentArg: Readonly<IControllerCredentialRelocationIntent>,
@@ -729,6 +732,14 @@ const closeKernelStoreConfirmed = async (
 
 const retainedEmbeddedDatabaseMigrations = new Set<EmbeddedDatabaseLocationMigration>();
 const retainedMigratedLocalSmartDbs = new Set<plugins.smartdb.LocalSmartDb>();
+const retainedCredentialKernelStores = new Set<plugins.smartsecret.SmartSecretKernelStore>();
+
+const drainRetainedCredentialKernelStores = async (): Promise<void> => {
+  for (const kernelStore of retainedCredentialKernelStores) {
+    await closeKernelStoreConfirmed(kernelStore);
+    retainedCredentialKernelStores.delete(kernelStore);
+  }
+};
 
 const stopLocalSmartDbConfirmed = async (
   localDbArg: plugins.smartdb.LocalSmartDb,
@@ -829,6 +840,7 @@ const defaultRelocateCredentialStore = async (
   signalArg?: AbortSignal,
 ): Promise<void> => {
   signalArg?.throwIfAborted();
+  await drainRetainedCredentialKernelStores();
   let kernelStore: plugins.smartsecret.SmartSecretKernelStore | undefined;
   let sealedStore: plugins.smartsecret.SmartSecretSealedFileStore | undefined;
   let operationError: unknown;
@@ -836,6 +848,7 @@ const defaultRelocateCredentialStore = async (
     kernelStore = await plugins.smartsecret.SmartSecretKernelStore.create({
       service: intentArg.service,
     });
+    retainedCredentialKernelStores.add(kernelStore);
     signalArg?.throwIfAborted();
     sealedStore = await plugins.smartsecret.SmartSecretSealedFileStore.relocate({
       kernelStore,
@@ -864,6 +877,7 @@ const defaultRelocateCredentialStore = async (
   if (kernelStore) {
     try {
       await closeKernelStoreConfirmed(kernelStore);
+      retainedCredentialKernelStores.delete(kernelStore);
     } catch (errorArg) {
       operationError = operationError
         ? new AggregateError(
@@ -952,6 +966,56 @@ export class ControllerDataRootMigrationRunner {
     return { directoryPath: this.newRoot, createDirectory: true };
   }
 
+  /** Validates all authoritative v2 state without creating roots, locks, or journals. */
+  public async preflight(): Promise<void> {
+    this.options.signal?.throwIfAborted();
+    if (this.options.invokerPid !== process.pid) {
+      throw new Error('Controller data-root migration invoker PID is not the current process.');
+    }
+    if (!await this.validateCommonParent(false)) return;
+    const journal = await this.readJournal();
+    if (journal) {
+      if (journal.phase === 'target-committed') {
+        this.assertJournalRootBinding(journal);
+        await this.assertTargetIdentity(journal);
+        await this.assertTargetMarker(journal.targetCreationNonce);
+        return;
+      }
+      this.assertJournalMatches(journal);
+      await this.assertSourceIdentity(journal);
+      await this.inspectLegacyRootInventory(journal);
+      await this.validateCredentialLocations(journal, false);
+      if (journal.phase !== 'legacy-authoritative') {
+        await this.inspectTargetStagingInventory(journal);
+      }
+      return;
+    }
+    const sourceIdentity = await readDirectoryIdentity(this.oldRoot, 'Legacy controller root');
+    const targetIdentity = await readDirectoryIdentity(
+      this.newRoot,
+      'Target controller root',
+      0o700,
+    );
+    const targetContainsPreservedDatabase = targetIdentity
+      ? await this.targetContainsOnlyPreservedDatabase()
+      : false;
+    if (sourceIdentity && targetIdentity && !targetContainsPreservedDatabase) {
+      throw new Error('Both unjournaled controller product roots exist; refusing to choose authority.');
+    }
+    if (sourceIdentity) {
+      await this.inspectLegacyRootInventory();
+      await this.initialCredentialTuples(true);
+      await this.initialDatabaseJournal();
+      return;
+    }
+    if (targetIdentity) {
+      const entries = await this.readBoundedDirectory(this.newRoot, 'Target controller root');
+      if (entries.length !== 0 && !targetContainsPreservedDatabase) {
+        throw new Error('An unjournaled target controller root is not empty.');
+      }
+    }
+  }
+
   public async run(): Promise<IControllerDataRootMigrationResult> {
     this.options.signal?.throwIfAborted();
     if (this.options.invokerPid !== process.pid) {
@@ -1029,6 +1093,23 @@ export class ControllerDataRootMigrationRunner {
           requireString(value.legacyEmbeddedDataDirectory, 'Historical embedded database path'),
           'Historical embedded database path',
         );
+    if (this.options.preserveEmbeddedDataDirectory === true) {
+      if (historicalDirectory !== null) {
+        throw new Error('An explicit embedded database override must not include a historical path.');
+      }
+      return {
+        mode: 'override',
+        configurationDigestSha256: databaseConfigurationDigest({
+          mode: 'override',
+          mongoDbName,
+          embeddedDataDirectory,
+        }),
+        oldDirectory: null,
+        targetDirectory: null,
+        historicalDirectory: null,
+        effectiveConfig: { mongoDbName, embeddedDataDirectory },
+      };
+    }
     if (embeddedDataDirectory !== oldDirectory && embeddedDataDirectory !== targetDirectory) {
       if (historicalDirectory !== null) {
         throw new Error('An explicit embedded database override must not include a historical path.');
@@ -1318,6 +1399,24 @@ export class ControllerDataRootMigrationRunner {
     };
   }
 
+  private preservedTargetDatabasePath(): string | undefined {
+    if (
+      this.databaseConfiguration.mode !== 'override'
+      || this.databaseConfiguration.effectiveConfig.embeddedDataDirectory
+        !== plugins.path.join(this.newRoot, 'smartdb')
+    ) return undefined;
+    return this.databaseConfiguration.effectiveConfig.embeddedDataDirectory;
+  }
+
+  private async targetContainsOnlyPreservedDatabase(): Promise<boolean> {
+    const databasePath = this.preservedTargetDatabasePath();
+    if (!databasePath) return false;
+    const names = await this.readBoundedDirectory(this.newRoot, 'Target controller root');
+    if (names.length !== 1 || names[0] !== 'smartdb') return false;
+    await this.requirePrivateNode(databasePath, 'directory', 'Explicit target database override');
+    return true;
+  }
+
   private async assertNoConflictingWriters(
     invokerIdentityArg: IControllerProcessIdentity,
   ): Promise<void> {
@@ -1349,13 +1448,16 @@ export class ControllerDataRootMigrationRunner {
   private async createInitialJournal(): Promise<IControllerDataRootMigrationJournal> {
     const sourceIdentity = await readDirectoryIdentity(this.oldRoot, 'Legacy controller root');
     const targetIdentity = await readDirectoryIdentity(this.newRoot, 'Target controller root', 0o700);
-    if (sourceIdentity && targetIdentity) {
+    const targetContainsPreservedDatabase = targetIdentity
+      ? await this.targetContainsOnlyPreservedDatabase()
+      : false;
+    if (sourceIdentity && targetIdentity && !targetContainsPreservedDatabase) {
       throw new Error('Both unjournaled controller product roots exist; refusing to choose authority.');
     }
     if (sourceIdentity) await this.inspectLegacyRootInventory();
     if (targetIdentity) {
       const entries = await this.readBoundedDirectory(this.newRoot, 'Target controller root');
-      if (entries.length !== 0) {
+      if (entries.length !== 0 && !targetContainsPreservedDatabase) {
         throw new Error('An unjournaled target controller root is not empty.');
       }
     }
@@ -1666,10 +1768,9 @@ export class ControllerDataRootMigrationRunner {
       if (!targetIdentity) throw new Error('Target controller root was not created.');
     }
     const entries = await this.readBoundedDirectory(this.newRoot, 'Target controller root');
-    if (
-      entries.length > 1
-      || (entries.length === 1 && entries[0] !== markerFileName)
-    ) {
+    const allowedEntries = new Set([markerFileName]);
+    if (this.preservedTargetDatabasePath()) allowedEntries.add('smartdb');
+    if (entries.some((entry) => !allowedEntries.has(entry))) {
       throw new Error('Legacy-authoritative retry found an unexpected staged target entry.');
     }
     await this.ensureTargetMarker(journalArg.targetCreationNonce);
@@ -2321,26 +2422,83 @@ export class ControllerDataRootMigrationRunner {
     let directoryChanged = false;
     for (const name of artifacts.temporaries) {
       const path = plugins.path.join(this.commonParent, name);
-      const snapshot = await this.readLockSnapshot(path);
-      if (!snapshot || snapshot.owner.uid !== currentUid()) {
-        throw new Error('Controller migration lock temporary artifact is invalid.');
+      const match = lockTemporarySuffixPattern.exec(name.slice(plugins.path.basename(this.lockPath).length));
+      if (!match) throw new Error(`Controller migration lock temporary name is invalid: ${name}`);
+      const uid = Number(match[1]);
+      const pid = Number(match[2]);
+      if (!Number.isSafeInteger(uid) || uid !== currentUid() || !Number.isSafeInteger(pid)) {
+        throw new Error(`Controller migration lock temporary binding is invalid: ${name}`);
       }
-      if (!await this.lockOwnerIsLive(snapshot.owner)) {
-        await unlinkExactPath(path, snapshot.identity);
+      let stats: plugins.fs.BigIntStats;
+      try {
+        stats = await plugins.fs.promises.lstat(path, { bigint: true });
+      } catch (errorArg) {
+        if (isMissingError(errorArg)) continue;
+        throw errorArg;
+      }
+      if (
+        !stats.isFile()
+        || stats.isSymbolicLink()
+        || stats.uid !== BigInt(currentUid())
+        || (stats.nlink !== 1n && stats.nlink !== 2n)
+        || Number(stats.mode & 0o777n) !== 0o600
+        || stats.size > BigInt(maximumLockBytes)
+      ) throw new Error(`Controller migration lock temporary artifact is unsafe: ${path}`);
+      const identity = pathIdentity(stats);
+      let snapshot: ILockSnapshot | undefined;
+      try {
+        snapshot = await this.readLockSnapshot(path);
+      } catch (errorArg) {
+        if (await readControllerProcessIdentity(pid)) continue;
+        await unlinkExactPath(path, identity);
         directoryChanged = true;
+        continue;
+      }
+      if (!snapshot) continue;
+      if (!identitiesEqual(snapshot.identity, identity)) {
+        throw new Error(`Controller migration lock temporary changed during inspection: ${path}`);
+      }
+      if (
+        snapshot.owner.uid !== uid
+        || snapshot.owner.pid !== pid
+        || snapshot.owner.nonce !== match[3]
+      ) {
+        throw new Error(`Controller migration lock temporary owner binding is invalid: ${path}`);
       }
+      if (await this.lockOwnerIsLive(snapshot.owner)) continue;
+      await unlinkExactPath(path, snapshot.identity);
+      directoryChanged = true;
     }
     for (const name of artifacts.guards) {
       const path = plugins.path.join(this.commonParent, name);
+      let stats: plugins.fs.BigIntStats;
+      try {
+        stats = await plugins.fs.promises.lstat(path, { bigint: true });
+      } catch (errorArg) {
+        if (isMissingError(errorArg)) continue;
+        throw errorArg;
+      }
+      if (
+        !stats.isFile()
+        || stats.isSymbolicLink()
+        || stats.uid !== BigInt(currentUid())
+        || (stats.nlink !== 1n && stats.nlink !== 2n)
+        || Number(stats.mode & 0o777n) !== 0o600
+        || stats.size < 2n
+        || stats.size > BigInt(maximumLockBytes)
+      ) throw new Error(`Controller migration lock guard is unsafe: ${path}`);
       const guard = await this.readLockSnapshot(path);
-      if (!guard) throw new Error('Controller migration lock guard disappeared during inspection.');
+      if (!guard) continue;
+      if (!identitiesEqual(guard.identity, pathIdentity(stats))) {
+        throw new Error('Controller migration lock guard changed during inspection.');
+      }
       if (guard.owner.uid !== currentUid()) {
         throw new Error('Controller migration lock guard belongs to another user.');
       }
-      if (!await this.lockOwnerIsLive(guard.owner)) {
-        await unlinkExactPath(path, guard.identity);
-        directoryChanged = true;
-      }
+      if (await this.lockOwnerIsLive(guard.owner)) continue;
+      if (Date.now() - Number(stats.ctimeMs) < lockGuardInitializationGraceMs) continue;
+      await unlinkExactPath(path, guard.identity);
+      directoryChanged = true;
     }
     if (directoryChanged) {
       await syncDirectory(this.commonParent);

package/ts_web/00_commitinfo_data.ts

@@ -3,6 +3,6 @@
  */
 export const commitinfo = {
   name: 'agl',
-  version: '21.0.2',
+  version: '22.0.0',
   description: 'Agent Gateway Layer for OpenCode and FlexHarness sessions'
 }