agl 1.0.17 → 21.0.2

package/ts_migration/classes.documentmigrationrunner.ts

@@ -0,0 +1,32 @@
+import * as plugins from '../ts/plugins.js';
+import { ProtocolV6DocumentMigration } from './v6_protocoldocumentmigration.js';
+import { ProtocolV18ResourceLayoutMigration } from './v18_resourcelayoutmigration.js';
+import { ProtocolV22ProjectFilesystemIdentityMigration } from './v22_projectfilesystemidentitymigration.js';
+
+interface IDocumentMigration {
+  run(): Promise<void>;
+}
+
+export class PostDatabaseDocumentMigrationRunner {
+  private readonly migrations: IDocumentMigration[];
+
+  constructor(private readonly database: plugins.smartdata.SmartdataDb | undefined) {
+    this.migrations = [
+      new ProtocolV6DocumentMigration(this.database),
+      new ProtocolV18ResourceLayoutMigration(this.database),
+    ];
+  }
+
+  public async run(): Promise<void> {
+    for (const migration of this.migrations) {
+      await migration.run();
+    }
+  }
+
+  public async runProjectFilesystemIdentity(controllerIdArg: string): Promise<void> {
+    await new ProtocolV22ProjectFilesystemIdentityMigration(
+      this.database,
+      controllerIdArg,
+    ).run();
+  }
+}

package/ts_migration/classes.flexmigrationrunner.ts

@@ -0,0 +1,33 @@
+import type { FlexStore } from '../ts/classes.flexstore.js';
+import { FlexHarnessV3MigrationRunner } from './v10_flexharnessv3migration.js';
+import { FlexProjectionOrderMigration } from './v13_flexprojectionorder.js';
+
+export interface IFlexPersistenceMigrationOptions {
+  store: FlexStore;
+  storageKeys: Iterable<string>;
+}
+
+export class FlexPersistenceMigrationRunner {
+  private readonly store: FlexStore;
+  private readonly storageKeys: string[];
+
+  constructor(optionsArg: IFlexPersistenceMigrationOptions) {
+    this.store = optionsArg.store;
+    this.storageKeys = [...new Set(optionsArg.storageKeys)].sort();
+  }
+
+  public async run(signalArg?: AbortSignal): Promise<void> {
+    await new FlexHarnessV3MigrationRunner({
+      store: this.store,
+      storageKeys: this.storageKeys,
+    }).run(signalArg);
+    await new FlexProjectionOrderMigration({
+      store: this.store,
+      storageKeys: this.storageKeys,
+    }).run(signalArg);
+    for (const storageKey of this.storageKeys) {
+      signalArg?.throwIfAborted();
+      await this.store.ensurePublicProjection(storageKey);
+    }
+  }
+}

package/ts_migration/classes.flexpostharnessmigrationrunner.ts

@@ -0,0 +1,22 @@
+import * as plugins from '../ts/plugins.js';
+import { FlexProjectManagementMigration } from './v19_flexprojectmanagement.js';
+
+export interface IFlexPostHarnessMigrationRunnerOptions<TScope> {
+  harness: plugins.flexharness.FlexHarness<TScope>;
+  scopeIds: Iterable<string>;
+}
+
+export class FlexPostHarnessMigrationRunner<TScope> {
+  private readonly migrations: Array<{ run(signalArg?: AbortSignal): Promise<void> }>;
+
+  constructor(optionsArg: IFlexPostHarnessMigrationRunnerOptions<TScope>) {
+    this.migrations = [new FlexProjectManagementMigration(optionsArg)];
+  }
+
+  public async run(signalArg?: AbortSignal): Promise<void> {
+    for (const migration of this.migrations) {
+      signalArg?.throwIfAborted();
+      await migration.run(signalArg);
+    }
+  }
+}

package/ts_migration/classes.migrationrunner.ts

@@ -0,0 +1,40 @@
+import {
+  EmbeddedDatabaseLocationMigration,
+  type IEmbeddedDatabaseLocationMigrationOptions,
+  type IEmbeddedDatabaseLocationMigrationResult,
+} from './v1_embeddeddatabaselocation.js';
+
+export class EmbeddedDatabaseMigrationRunner {
+  private readonly migrations: EmbeddedDatabaseLocationMigration[];
+
+  constructor(optionsArg: IEmbeddedDatabaseLocationMigrationOptions) {
+    this.migrations = [new EmbeddedDatabaseLocationMigration(optionsArg)];
+  }
+
+  public get hasOwnedResources(): boolean {
+    return this.migrations.some((migration) => migration.hasOwnedResources);
+  }
+
+  public async run(): Promise<IEmbeddedDatabaseLocationMigrationResult | undefined> {
+    let result: IEmbeddedDatabaseLocationMigrationResult | undefined;
+    for (const migration of this.migrations) {
+      result = await migration.run() ?? result;
+    }
+    return result;
+  }
+
+  public async stop(): Promise<void> {
+    const errors: unknown[] = [];
+    for (const migration of [...this.migrations].reverse()) {
+      try {
+        await migration.stop();
+      } catch (errorArg) {
+        errors.push(errorArg);
+      }
+    }
+    if (errors.length === 1) throw errors[0];
+    if (errors.length > 1) {
+      throw new AggregateError(errors, 'Embedded database migration runner cleanup failed.');
+    }
+  }
+}

package/ts_migration/index.ts

@@ -0,0 +1,12 @@
+export * from './classes.migrationrunner.js';
+export * from './classes.flexmigrationrunner.js';
+export * from './v1_embeddeddatabaselocation.js';
+export * from './v2_controllerdataroot.js';
+export * from './v10_flexharnessv3migration.js';
+export * from './v13_flexprojectionorder.js';
+export * from './v16_flexprovidercredentials.js';
+export * from './v18_resourcelayout.js';
+export * from './v18_resourcelayoutmigration.js';
+export * from './classes.flexpostharnessmigrationrunner.js';
+export * from './v19_flexprojectmanagement.js';
+export * from './v22_projectfilesystemidentitymigration.js';

package/ts_migration/v10_flexharnessv3migration.ts

@@ -0,0 +1,35 @@
+import * as plugins from '../ts/plugins.js';
+import type { FlexStore } from '../ts/classes.flexstore.js';
+
+export interface IFlexHarnessV3MigrationOptions {
+  store: FlexStore;
+  storageKeys: Iterable<string>;
+}
+
+export class FlexHarnessV3MigrationRunner {
+  private readonly store: FlexStore;
+  private readonly storageKeys: string[];
+
+  constructor(optionsArg: IFlexHarnessV3MigrationOptions) {
+    this.store = optionsArg.store;
+    this.storageKeys = [...new Set(optionsArg.storageKeys)].sort();
+  }
+
+  public async run(signalArg?: AbortSignal): Promise<void> {
+    const migrationStores = this.store.getLegacyMigrationStores();
+    for (const storageKey of this.storageKeys) {
+      signalArg?.throwIfAborted();
+      if (!await this.store.hasCompletedLegacyMigration(storageKey)) {
+        const legacySnapshot = await this.store.loadLegacy(storageKey);
+        if (legacySnapshot) {
+          await plugins.flexharnessMigration.migrateLegacyFlexHarnessSnapshot(
+            storageKey,
+            legacySnapshot,
+            migrationStores,
+          );
+          await this.store.completeLegacyMigration(storageKey);
+        }
+      }
+    }
+  }
+}

package/ts_migration/v13_flexprojectionorder.ts

@@ -0,0 +1,33 @@
+import type { FlexStore } from '../ts/classes.flexstore.js';
+
+export interface IFlexProjectionOrderMigrationOptions {
+  store: FlexStore;
+  storageKeys: Iterable<string>;
+}
+
+export class FlexProjectionOrderMigration {
+  private readonly store: FlexStore;
+  private readonly storageKeys: string[];
+
+  constructor(optionsArg: IFlexProjectionOrderMigrationOptions) {
+    this.store = optionsArg.store;
+    this.storageKeys = [...new Set(optionsArg.storageKeys)].sort();
+  }
+
+  public async run(signalArg?: AbortSignal): Promise<void> {
+    for (const storageKey of this.storageKeys) {
+      let cursor: Awaited<ReturnType<FlexStore['listProjectionOrderMigrationPage']>>[
+        'nextCursor'
+      ];
+      do {
+        signalArg?.throwIfAborted();
+        const page = await this.store.listProjectionOrderMigrationPage(storageKey, cursor);
+        for (const sessionId of page.sessionIds) {
+          signalArg?.throwIfAborted();
+          await this.store.rewriteProjectionSourceOrder(storageKey, sessionId);
+        }
+        cursor = page.nextCursor;
+      } while (cursor);
+    }
+  }
+}

package/ts_migration/v16_flexprovidercredentials.ts

@@ -0,0 +1,196 @@
+import * as plugins from '../ts/plugins.js';
+
+export const flexProviderCredentialStoreId = 'flex-provider-credentials.v1';
+export const flexProviderCredentialConnectionLimit = 512;
+
+export interface IFlexProviderCredentialStore {
+  getEntry(accountArg: string): Promise<Uint8Array | null>;
+  setEntry(accountArg: string, valueArg: Uint8Array): Promise<void>;
+  deleteEntry(accountArg: string): Promise<boolean>;
+}
+
+export interface IFlexProviderCredentialStorePair<
+  TKernelStore extends IFlexProviderCredentialStore = IFlexProviderCredentialStore,
+  TSealedStore extends IFlexProviderCredentialStore = IFlexProviderCredentialStore,
+> {
+  kernelStore: TKernelStore;
+  sealedStore: TSealedStore;
+}
+
+export interface IFlexProviderCredentialMigrationOptions<
+  TKernelStore extends IFlexProviderCredentialStore,
+  TSealedStore extends IFlexProviderCredentialStore,
+> {
+  stores: IFlexProviderCredentialStorePair<TKernelStore, TSealedStore>;
+  recreateStores: (
+    storesArg: IFlexProviderCredentialStorePair<TKernelStore, TSealedStore>,
+  ) => Promise<IFlexProviderCredentialStorePair<TKernelStore, TSealedStore>>;
+  connectionIds: Iterable<string>;
+}
+
+export type TFlexProviderCredentialMigrationErrorCode =
+  | 'CONNECTION_LIMIT_EXCEEDED'
+  | 'CREDENTIAL_CONFLICT'
+  | 'CREDENTIAL_INVALID'
+  | 'MUTATION_RECONCILIATION_FAILED';
+
+export class FlexProviderCredentialMigrationError extends Error {
+  constructor(public readonly code: TFlexProviderCredentialMigrationErrorCode) {
+    super(`Flex provider credential migration failed (${code}).`);
+    this.name = 'FlexProviderCredentialMigrationError';
+  }
+}
+
+const maximumMutationAttempts = 2;
+
+const credentialAccount = (connectionIdArg: string): string => `provider:${connectionIdArg}`;
+
+const isMutationOutcomeUnknown = (errorArg: unknown): boolean => (
+  errorArg instanceof plugins.smartsecret.SmartSecretKernelStoreError
+  || errorArg instanceof plugins.smartsecret.SmartSecretSealedFileStoreError
+) && errorArg.code === 'MUTATION_OUTCOME_UNKNOWN';
+
+const credentialBytesEqual = (leftArg: Uint8Array, rightArg: Uint8Array): boolean =>
+  leftArg.byteLength === rightArg.byteLength
+  && plugins.crypto.timingSafeEqual(leftArg, rightArg);
+
+const validateCredentialBytes = (bytesArg: Uint8Array): void => {
+  try {
+    const credential = plugins.smartaiProviders.parseProviderCredential(bytesArg);
+    if (credential.kind !== 'chatgptOAuth' || credential.providerId !== 'openai') {
+      throw new FlexProviderCredentialMigrationError('CREDENTIAL_INVALID');
+    }
+  } catch (errorArg) {
+    if (errorArg instanceof FlexProviderCredentialMigrationError) throw errorArg;
+    throw new FlexProviderCredentialMigrationError('CREDENTIAL_INVALID');
+  }
+};
+
+const readValidatedCredential = async (
+  storeArg: IFlexProviderCredentialStore,
+  accountArg: string,
+): Promise<Uint8Array | null> => {
+  const bytes = await storeArg.getEntry(accountArg);
+  if (!bytes) return null;
+  try {
+    validateCredentialBytes(bytes);
+    return bytes;
+  } catch (errorArg) {
+    bytes.fill(0);
+    throw errorArg;
+  }
+};
+
+const deleteLegacyCredentialEntry = async <
+  TKernelStore extends IFlexProviderCredentialStore,
+  TSealedStore extends IFlexProviderCredentialStore,
+>(
+  optionsArg: Pick<
+    IFlexProviderCredentialMigrationOptions<TKernelStore, TSealedStore>,
+    'stores' | 'recreateStores'
+  >,
+  accountArg: string,
+  expectedBytesArg: Uint8Array,
+): Promise<IFlexProviderCredentialStorePair<TKernelStore, TSealedStore>> => {
+  let stores = optionsArg.stores;
+  let requiresConfirmedRetry = false;
+  for (let attempt = 0; attempt < maximumMutationAttempts; attempt += 1) {
+    let outcomeWasUnknown = false;
+    try {
+      await stores.kernelStore.deleteEntry(accountArg);
+    } catch (errorArg) {
+      if (!isMutationOutcomeUnknown(errorArg)) throw errorArg;
+      stores = await optionsArg.recreateStores(stores);
+      outcomeWasUnknown = true;
+      requiresConfirmedRetry = true;
+    }
+    const readback = await readValidatedCredential(stores.kernelStore, accountArg);
+    try {
+      if (!readback) {
+        if (!requiresConfirmedRetry || !outcomeWasUnknown) return stores;
+        continue;
+      }
+      if (!credentialBytesEqual(readback, expectedBytesArg)) {
+        throw new FlexProviderCredentialMigrationError('CREDENTIAL_CONFLICT');
+      }
+      if (!outcomeWasUnknown || attempt === maximumMutationAttempts - 1) {
+        throw new FlexProviderCredentialMigrationError('MUTATION_RECONCILIATION_FAILED');
+      }
+    } finally {
+      readback?.fill(0);
+    }
+  }
+  throw new FlexProviderCredentialMigrationError('MUTATION_RECONCILIATION_FAILED');
+};
+
+export const migrateFlexProviderCredentials = async <
+  TKernelStore extends IFlexProviderCredentialStore,
+  TSealedStore extends IFlexProviderCredentialStore,
+>(
+  optionsArg: IFlexProviderCredentialMigrationOptions<TKernelStore, TSealedStore>,
+  signalArg?: AbortSignal,
+): Promise<IFlexProviderCredentialStorePair<TKernelStore, TSealedStore>> => {
+  const connectionIds = [...new Set(optionsArg.connectionIds)].sort();
+  if (connectionIds.length > flexProviderCredentialConnectionLimit) {
+    throw new FlexProviderCredentialMigrationError('CONNECTION_LIMIT_EXCEEDED');
+  }
+  let stores = optionsArg.stores;
+  for (const connectionId of connectionIds) {
+    signalArg?.throwIfAborted();
+    const account = credentialAccount(connectionId);
+    let sourceBytes: Uint8Array | null = null;
+    let sealedBytes: Uint8Array | null = null;
+    try {
+      sourceBytes = await readValidatedCredential(stores.kernelStore, account);
+      sealedBytes = await readValidatedCredential(stores.sealedStore, account);
+      if (!sourceBytes) continue;
+      if (sealedBytes && !credentialBytesEqual(sourceBytes, sealedBytes)) {
+        throw new FlexProviderCredentialMigrationError('CREDENTIAL_CONFLICT');
+      }
+      if (!sealedBytes) {
+        let copied = false;
+        let requiresConfirmedRetry = false;
+        for (let attempt = 0; attempt < maximumMutationAttempts; attempt += 1) {
+          let outcomeWasUnknown = false;
+          try {
+            await stores.sealedStore.setEntry(account, sourceBytes);
+          } catch (errorArg) {
+            if (!isMutationOutcomeUnknown(errorArg)) throw errorArg;
+            stores = await optionsArg.recreateStores(stores);
+            outcomeWasUnknown = true;
+            requiresConfirmedRetry = true;
+          }
+          const readback = await readValidatedCredential(stores.sealedStore, account);
+          try {
+            if (readback && credentialBytesEqual(readback, sourceBytes)) {
+              if (!requiresConfirmedRetry || !outcomeWasUnknown) {
+                copied = true;
+                break;
+              }
+              continue;
+            }
+            if (readback) {
+              throw new FlexProviderCredentialMigrationError('CREDENTIAL_CONFLICT');
+            }
+            if (!outcomeWasUnknown || attempt === maximumMutationAttempts - 1) {
+              throw new FlexProviderCredentialMigrationError('MUTATION_RECONCILIATION_FAILED');
+            }
+          } finally {
+            readback?.fill(0);
+          }
+        }
+        if (!copied) {
+          throw new FlexProviderCredentialMigrationError('MUTATION_RECONCILIATION_FAILED');
+        }
+      }
+      stores = await deleteLegacyCredentialEntry({
+        stores,
+        recreateStores: optionsArg.recreateStores,
+      }, account, sourceBytes);
+    } finally {
+      sourceBytes?.fill(0);
+      sealedBytes?.fill(0);
+    }
+  }
+  return stores;
+};

package/ts_migration/v18_resourcelayout.ts

@@ -0,0 +1,55 @@
+import type {
+  IControllerSessionGroupsDocument,
+} from '../ts/interfaces.projects.js';
+import type { TControllerSessionId } from '../ts_interfaces/index.js';
+
+type TObject = Record<string, unknown>;
+
+export interface IProtocolV18LayoutMigrationResult {
+  document: IControllerSessionGroupsDocument;
+  migrated: boolean;
+}
+
+export interface IProtocolV18LegacySessionGroup {
+  id: string;
+  name: string;
+  sessionIds: Array<TControllerSessionId | { harnessId: 'controller'; nativeId: string }>;
+}
+
+export interface IProtocolV18LegacySessionGroupsDocument
+extends Omit<IControllerSessionGroupsDocument, 'groups' | 'ungroupedSessionIds'> {
+  groups: IProtocolV18LegacySessionGroup[];
+  ungroupedSessionIds?: Array<TControllerSessionId | { harnessId: 'controller'; nativeId: string }>;
+}
+
+const isPlainObject = (valueArg: unknown): valueArg is TObject =>
+  typeof valueArg === 'object' && valueArg !== null && !Array.isArray(valueArg);
+
+const isLegacyTerminalId = (valueArg: unknown): boolean =>
+  isPlainObject(valueArg) && valueArg.harnessId === 'controller';
+
+export const migrateProtocolV18SessionLayoutDocument = (
+  valueArg: IProtocolV18LegacySessionGroupsDocument,
+): IProtocolV18LayoutMigrationResult => {
+  const groups = valueArg.groups.map((group) => ({
+    ...group,
+    sessionIds: group.sessionIds.filter(
+      (sessionId): sessionId is TControllerSessionId => !isLegacyTerminalId(sessionId),
+    ),
+  }));
+  const ungroupedSessionIds = (valueArg.ungroupedSessionIds ?? [])
+    .filter((sessionId): sessionId is TControllerSessionId => !isLegacyTerminalId(sessionId));
+  const migrated = groups.some((group, index) => (
+    group.sessionIds.length !== valueArg.groups[index].sessionIds.length
+  )) || ungroupedSessionIds.length !== (valueArg.ungroupedSessionIds ?? []).length;
+  if (!migrated) return { document: valueArg as IControllerSessionGroupsDocument, migrated: false };
+  return {
+    document: {
+      ...valueArg,
+      groups,
+      ungroupedSessionIds,
+      revision: (valueArg.revision ?? 0) + 1,
+    },
+    migrated: true,
+  };
+};

package/ts_migration/v18_resourcelayoutmigration.ts

@@ -0,0 +1,68 @@
+import * as plugins from '../ts/plugins.js';
+import {
+  assertControllerSessionGroupsDocument,
+  ControllerSessionGroupsModel,
+} from '../ts/classes.authmodels.js';
+import { AuthError } from '../ts/interfaces.auth.js';
+import type { IControllerSessionGroupsDocument } from '../ts/interfaces.projects.js';
+import { migrateProtocolV18SessionLayoutDocument } from './v18_resourcelayout.js';
+import type { IProtocolV18LegacySessionGroupsDocument } from './v18_resourcelayout.js';
+
+const migrationPageLimit = 128;
+
+const persistedBodyFromRawDocument = (
+  valueArg: Record<string, unknown>,
+): IProtocolV18LegacySessionGroupsDocument => {
+  const body = { ...valueArg };
+  delete body._id;
+  delete body._smartdataRevision;
+  return body as unknown as IProtocolV18LegacySessionGroupsDocument;
+};
+
+export class ProtocolV18ResourceLayoutMigration {
+  constructor(private readonly database: plugins.smartdata.SmartdataDb | undefined) {}
+
+  public async run(): Promise<void> {
+    if (!this.database) {
+      throw new AuthError('not_initialized', 'The authentication store database is unavailable.');
+    }
+    const collection = ControllerSessionGroupsModel.collection.mongoDbCollection;
+    let lastId: plugins.smartdata.TStoredDocument<IControllerSessionGroupsDocument>['_id'] | undefined;
+    while (true) {
+      const cursor = collection.find(lastId ? { _id: { $gt: lastId } } : {})
+        .sort({ _id: 1 })
+        .limit(migrationPageLimit);
+      let documents: Awaited<ReturnType<typeof cursor.toArray>>;
+      try {
+        documents = await cursor.toArray();
+      } finally {
+        await cursor.close();
+      }
+      for (const raw of documents) {
+        const migration = migrateProtocolV18SessionLayoutDocument(persistedBodyFromRawDocument(raw));
+        assertControllerSessionGroupsDocument(migration.document);
+        if (!migration.migrated) continue;
+        const selector = raw._smartdataRevision === undefined
+          ? { _id: raw._id, _smartdataRevision: { $exists: false } }
+          : { _id: raw._id, _smartdataRevision: raw._smartdataRevision };
+        const replaced = await collection.findOneAndReplace(
+          selector,
+          { _id: raw._id, ...migration.document, _smartdataRevision: plugins.crypto.randomUUID() },
+          { returnDocument: 'after', includeResultMetadata: false, upsert: false },
+        );
+        if (!replaced) {
+          const concurrent = await collection.findOne({ _id: raw._id });
+          if (!concurrent) throw new AuthError('concurrent_change', 'The resource layout migration changed concurrently.');
+          const remaining = migrateProtocolV18SessionLayoutDocument(persistedBodyFromRawDocument(concurrent));
+          assertControllerSessionGroupsDocument(remaining.document);
+          if (remaining.migrated) {
+            throw new AuthError('concurrent_change', 'The resource layout migration changed concurrently.');
+          }
+        }
+      }
+      const last = documents.at(-1);
+      if (!last || documents.length < migrationPageLimit) break;
+      lastId = last._id;
+    }
+  }
+}

package/ts_migration/v19_flexprojectmanagement.ts

@@ -0,0 +1,28 @@
+import * as plugins from '../ts/plugins.js';
+
+export interface IFlexProjectManagementMigrationOptions<TScope> {
+  harness: plugins.flexharness.FlexHarness<TScope>;
+  scopeIds: Iterable<string>;
+}
+
+export class FlexProjectManagementMigration<TScope> {
+  private readonly harness: plugins.flexharness.FlexHarness<TScope>;
+  private readonly scopeIds: string[];
+
+  constructor(optionsArg: IFlexProjectManagementMigrationOptions<TScope>) {
+    this.harness = optionsArg.harness;
+    this.scopeIds = [...new Set(optionsArg.scopeIds)].sort();
+  }
+
+  public async run(signalArg?: AbortSignal): Promise<void> {
+    for (const scopeId of this.scopeIds) {
+      signalArg?.throwIfAborted();
+      // listSessions durably repairs legacy session generations before PM state is loaded.
+      const sessions = await this.harness.listSessions(scopeId);
+      for (const session of sessions) {
+        signalArg?.throwIfAborted();
+        await this.harness.getProjectState(scopeId, session.sessionId);
+      }
+    }
+  }
+}