@dxos/migrations 0.8.3 → 0.8.4-main.05e74ebcff

package/package.json

@@ -1,41 +1,44 @@
 {
   "name": "@dxos/migrations",
-  "version": "0.8.3",
+  "version": "0.8.4-main.05e74ebcff",
   "description": "",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
-  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dxos/dxos"
+  },
+  "license": "FSL-1.1-Apache-2.0",
   "author": "info@dxos.org",
-  "sideEffects": true,
+  "sideEffects": false,
   "type": "module",
   "exports": {
     ".": {
+      "source": "./src/index.ts",
       "types": "./dist/types/src/index.d.ts",
       "browser": "./dist/lib/browser/index.mjs",
       "node": "./dist/lib/node-esm/index.mjs"
     }
   },
   "types": "dist/types/src/index.d.ts",
-  "typesVersions": {
-    "*": {}
-  },
   "files": [
     "dist",
     "src"
   ],
   "dependencies": {
-    "@automerge/automerge": "3.0.0-beta.4",
-    "@automerge/automerge-repo": "2.0.1",
-    "@dxos/client": "0.8.3",
-    "@dxos/echo-db": "0.8.3",
-    "@dxos/echo-schema": "0.8.3",
-    "@dxos/invariant": "0.8.3",
-    "@dxos/log": "0.8.3",
-    "@dxos/echo-protocol": "0.8.3",
-    "@dxos/protocols": "0.8.3",
-    "@dxos/util": "0.8.3"
+    "@automerge/automerge": "3.2.6",
+    "@automerge/automerge-repo": "2.6.0-subduction.17",
+    "@effect-atom/atom": "^0.5.1",
+    "@dxos/echo-db": "0.8.4-main.05e74ebcff",
+    "@dxos/echo-protocol": "0.8.4-main.05e74ebcff",
+    "@dxos/invariant": "0.8.4-main.05e74ebcff",
+    "@dxos/keys": "0.8.4-main.05e74ebcff",
+    "@dxos/log": "0.8.4-main.05e74ebcff",
+    "@dxos/client": "0.8.4-main.05e74ebcff",
+    "@dxos/util": "0.8.4-main.05e74ebcff",
+    "@dxos/protocols": "0.8.4-main.05e74ebcff",
+    "@dxos/echo": "0.8.4-main.05e74ebcff"
   },
-  "devDependencies": {},
   "publishConfig": {
     "access": "public"
   }

package/src/migration-builder.ts

@@ -2,22 +2,17 @@
 // Copyright 2024 DXOS.org
 //
 
-import { next as am, type Doc } from '@automerge/automerge';
+import { next as A, type Doc } from '@automerge/automerge';
 import { type AnyDocumentId, type DocumentId } from '@automerge/automerge-repo';
-import { type Schema } from 'effect';
+import type * as Schema from 'effect/Schema';
 
 import { type Space } from '@dxos/client/echo';
 import { CreateEpochRequest } from '@dxos/client/halo';
-import { ObjectCore, migrateDocument, type DocHandleProxy, type RepoProxy } from '@dxos/echo-db';
-import {
-  Reference,
-  SpaceDocVersion,
-  encodeReference,
-  type DatabaseDirectory,
-  type ObjectStructure,
-} from '@dxos/echo-protocol';
-import { requireTypeReference } from '@dxos/echo-schema';
+import { type DocHandleProxy, ObjectCore, type RepoProxy, migrateDocument } from '@dxos/echo-db';
+import { type DatabaseDirectory, EncodedReference, type ObjectStructure, SpaceDocVersion } from '@dxos/echo-protocol';
+import { getSchemaDXN } from '@dxos/echo/internal';
 import { invariant } from '@dxos/invariant';
+import { DXN } from '@dxos/keys';
 import { type MaybePromise } from '@dxos/util';
 
 /*
@@ -50,9 +45,9 @@ export class MigrationBuilder {
   private _newRoot?: DocHandleProxy<DatabaseDirectory> = undefined;
 
   constructor(private readonly _space: Space) {
-    this._repo = this._space.db.coreDatabase._repo;
+    this._repo = this._space.internal.db.coreDatabase._repo;
     // TODO(wittjosiah): Accessing private API.
-    this._rootDoc = (this._space.db.coreDatabase as any)._automergeDocLoader
+    this._rootDoc = (this._space.internal.db.coreDatabase as any)._automergeDocLoader
       .getSpaceRootDocHandle()
       .doc() as Doc<DatabaseDirectory>;
   }
@@ -91,7 +86,7 @@ export class MigrationBuilder {
       objects: {
         [id]: {
           system: {
-            type: encodeReference(requireTypeReference(schema)),
+            type: EncodedReference.fromDXN(getSchemaDXN(schema)!),
           },
           data: props,
           meta: {
@@ -101,27 +96,29 @@ export class MigrationBuilder {
       },
     };
     const migratedDoc = migrateDocument(oldHandle.doc() as Doc<DatabaseDirectory>, newState);
-    const newHandle = this._repo.import<DatabaseDirectory>(am.save(migratedDoc));
+    const newHandle = this._repo.import<DatabaseDirectory>(A.save(migratedDoc));
+    await newHandle.whenReady();
+    invariant(newHandle.url, 'Migrated document URL not available after whenReady');
     this._newLinks[id] = newHandle.url;
-    this._addHandleToFlushList(newHandle);
+    this._addHandleToFlushList(newHandle.documentId!);
   }
 
   async addObject(schema: Schema.Schema.AnyNoContext, props: any): Promise<string> {
-    const core = this._createObject({ schema, props });
+    const core = await this._createObject({ schema, props });
     return core.id;
   }
 
   createReference(id: string) {
-    return encodeReference(Reference.localObjectReference(id));
+    return EncodedReference.fromDXN(DXN.fromLocalObjectId(id));
   }
 
   deleteObject(id: string): void {
     this._deleteObjects.push(id);
   }
 
-  changeProperties(changeFn: (properties: ObjectStructure) => void): void {
+  async changeProperties(changeFn: (properties: ObjectStructure) => void): Promise<void> {
     if (!this._newRoot) {
-      this._buildNewRoot();
+      await this._buildNewRoot();
     }
     invariant(this._newRoot, 'New root not created');
 
@@ -129,7 +126,8 @@ export class MigrationBuilder {
       const propertiesStructure = doc.objects?.[this._space.properties.id];
       propertiesStructure && changeFn(propertiesStructure);
     });
-    this._addHandleToFlushList(this._newRoot);
+    await this._newRoot.whenReady();
+    this._addHandleToFlushList(this._newRoot.documentId!);
   }
 
   /**
@@ -137,13 +135,14 @@ export class MigrationBuilder {
    */
   async _commit(): Promise<void> {
     if (!this._newRoot) {
-      this._buildNewRoot();
+      await this._buildNewRoot();
     }
     invariant(this._newRoot, 'New root not created');
 
     await this._space.db.flush();
 
     // Create new epoch.
+    invariant(this._newRoot.url, 'New root URL not available');
     await this._space.internal.createEpoch({
       migration: CreateEpochRequest.Migration.REPLACE_AUTOMERGE_ROOT,
       automergeRootUrl: this._newRoot.url,
@@ -161,14 +160,14 @@ export class MigrationBuilder {
     return docHandle;
   }
 
-  private _buildNewRoot(): void {
+  private async _buildNewRoot(): Promise<void> {
     const links = { ...(this._rootDoc.links ?? {}) };
     for (const id of this._deleteObjects) {
       delete links[id];
     }
 
     for (const [id, url] of Object.entries(this._newLinks)) {
-      links[id] = new am.RawString(url);
+      links[id] = new A.RawString(url);
     }
 
     this._newRoot = this._repo.create<DatabaseDirectory>({
@@ -179,10 +178,11 @@ export class MigrationBuilder {
       objects: this._rootDoc.objects,
       links,
     });
-    this._addHandleToFlushList(this._newRoot);
+    await this._newRoot.whenReady();
+    this._addHandleToFlushList(this._newRoot.documentId!);
   }
 
-  private _createObject({
+  private async _createObject({
     id,
     schema,
     props,
@@ -190,14 +190,14 @@ export class MigrationBuilder {
     id?: string;
     schema: Schema.Schema.AnyNoContext;
     props: any;
-  }): ObjectCore {
+  }): Promise<ObjectCore> {
     const core = new ObjectCore();
     if (id) {
       core.id = id;
     }
 
     core.initNewObject(props);
-    core.setType(requireTypeReference(schema));
+    core.setType(EncodedReference.fromDXN(getSchemaDXN(schema)!));
     const newHandle = this._repo.create<DatabaseDirectory>({
       version: SpaceDocVersion.CURRENT,
       access: {
@@ -207,13 +207,14 @@ export class MigrationBuilder {
         [core.id]: core.getDoc() as ObjectStructure,
       },
     });
-    this._newLinks[core.id] = newHandle.url;
-    this._addHandleToFlushList(newHandle);
+    await newHandle.whenReady();
+    this._newLinks[core.id] = newHandle.url!;
+    this._addHandleToFlushList(newHandle.documentId!);
 
     return core;
   }
 
-  private _addHandleToFlushList(handle: DocHandleProxy<any>): void {
-    this._flushIds.push(handle.documentId);
+  private _addHandleToFlushList(id: DocumentId): void {
+    this._flushIds.push(id);
   }
 }

package/src/migrations.test.ts

@@ -5,9 +5,10 @@
 import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'vitest';
 
 import { Client } from '@dxos/client';
-import { live, Filter, type Space } from '@dxos/client/echo';
+import { type Space } from '@dxos/client/echo';
 import { TestBuilder } from '@dxos/client/testing';
-import { Expando } from '@dxos/echo-schema';
+import { Filter, Obj } from '@dxos/echo';
+import { TestSchema } from '@dxos/echo/testing';
 
 import { Migrations } from './migrations';
 
@@ -15,17 +16,17 @@ Migrations.define('test', [
   {
     version: '1970-01-01',
     next: async ({ builder }) => {
-      await builder.addObject(Expando, { namespace: 'test', count: 1 });
+      await builder.addObject(TestSchema.Expando, { namespace: 'test', count: 1 });
     },
   },
   {
     version: '1970-01-02',
     next: async ({ space, builder }) => {
-      // TODO(dmaretskyi): Is this intended to query only expando objects? Change to `Filter.type(Expando, { namespace: 'test' })`
-      const { objects } = await space.db.query(Filter._props<any>({ namespace: 'test' })).run();
+      // TODO(dmaretskyi): Is this intended to query only expando objects? Change to `Filter.type(TestSchema.Expando, { namespace: 'test' })`
+      const objects = await space.db.query(Filter.props<any>({ namespace: 'test' })).run();
       for (const object of objects) {
         await builder.migrateObject(object.id, ({ data }) => ({
-          schema: Expando,
+          schema: TestSchema.Expando,
           props: { namespace: data.namespace, count: 2 },
         }));
       }
@@ -34,11 +35,11 @@ Migrations.define('test', [
   {
     version: '1970-01-03',
     next: async ({ space, builder }) => {
-      // TODO(dmaretskyi): Is this intended to query only expando objects? Change to `Filter.type(Expando, { namespace: 'test' })`
-      const { objects } = await space.db.query(Filter._props<any>({ namespace: 'test' })).run();
+      // TODO(dmaretskyi): Is this intended to query only expando objects? Change to `Filter.type(TestSchema.Expando, { namespace: 'test' })`
+      const objects = await space.db.query(Filter.props<any>({ namespace: 'test' })).run();
       for (const object of objects) {
         await builder.migrateObject(object.id, ({ data }) => ({
-          schema: Expando,
+          schema: TestSchema.Expando,
           props: { namespace: data.namespace, count: data.count * 3 },
         }));
       }
@@ -46,7 +47,8 @@ Migrations.define('test', [
   },
 ]);
 
-describe('Migrations', () => {
+// Flaky. We wanna depreacate and rewrite migration builder anyway.
+describe.skip('Migrations', () => {
   let client: Client;
   let space: Space;
 
@@ -67,32 +69,38 @@ describe('Migrations', () => {
 
   test('if no migrations have been run before, runs all migrations', async () => {
     await Migrations.migrate(space);
-    const { objects } = await space.db.query(Filter.type(Expando, { namespace: 'test' })).run();
+    const objects = await space.db.query(Filter.type(TestSchema.Expando, { namespace: 'test' })).run();
     expect(objects).to.have.length(1);
     expect(objects[0].count).to.equal(6);
     expect(space.properties['test.version']).to.equal('1970-01-03');
   });
 
   test('if some migrations have been run before, runs only the remaining migrations', async () => {
-    space.properties['test.version'] = '1970-01-02';
-    space.db.add(live(Expando, { namespace: 'test', count: 5 }));
+    Obj.update(space.properties, (obj) => {
+      obj['test.version'] = '1970-01-02';
+    });
+    await space.db.graph.schemaRegistry.register([TestSchema.Expando]);
+    space.db.add(Obj.make(TestSchema.Expando, { namespace: 'test', count: 5 }));
+    await space.db.flush();
     await Migrations.migrate(space);
-    const { objects } = await space.db.query(Filter.type(Expando, { namespace: 'test' })).run();
+    const objects = await space.db.query(Filter.type(TestSchema.Expando, { namespace: 'test' })).run();
     expect(objects).to.have.length(1);
     expect(objects[0].count).to.equal(15);
     expect(space.properties['test.version']).to.equal('1970-01-03');
   });
 
   test('if all migrations have been run before, does nothing', async () => {
-    space.properties['test.version'] = '1970-01-03';
+    Obj.update(space.properties, (obj) => {
+      obj['test.version'] = '1970-01-03';
+    });
     await Migrations.migrate(space);
-    const { objects } = await space.db.query(Filter.type(Expando, { namespace: 'test' })).run();
+    const objects = await space.db.query(Filter.type(TestSchema.Expando, { namespace: 'test' })).run();
     expect(objects).to.have.length(0);
   });
 
   test('if target version is specified, runs only the migrations up to that version', async () => {
     await Migrations.migrate(space, '1970-01-02');
-    const { objects } = await space.db.query(Filter.type(Expando, { namespace: 'test' })).run();
+    const objects = await space.db.query(Filter.type(TestSchema.Expando, { namespace: 'test' })).run();
     expect(objects).to.have.length(1);
     expect(objects[0].count).to.equal(2);
     expect(space.properties['test.version']).to.equal('1970-01-02');

package/src/migrations.ts

@@ -2,7 +2,10 @@
 // Copyright 2023 DXOS.org
 //
 
-import { type Space, live, SpaceState } from '@dxos/client/echo';
+import { Atom } from '@effect-atom/atom';
+import * as Registry from '@effect-atom/atom/Registry';
+
+import { type Space, SpaceState } from '@dxos/client/echo';
 import { invariant } from '@dxos/invariant';
 import { type MaybePromise } from '@dxos/util';
 
@@ -21,7 +24,8 @@ export type Migration = {
 export class Migrations {
   static namespace?: string;
   static migrations: Migration[] = [];
-  private static _state = live<{ running: string[] }>({ running: [] });
+  private static _registry = Registry.make();
+  private static _stateAtom = Atom.make<{ running: string[] }>({ running: [] }).pipe(Atom.keepAlive);
 
   static get versionProperty() {
     return this.namespace && `${this.namespace}.version`;
@@ -32,7 +36,8 @@ export class Migrations {
   }
 
   static running(space: Space): boolean {
-    return this._state.running.includes(space.key.toHex());
+    const state = this._registry.get(this._stateAtom);
+    return state.running.includes(space.key.toHex());
   }
 
   static define(namespace: string, migrations: Migration[]): void {
@@ -52,20 +57,23 @@ export class Migrations {
       return false;
     }
 
-    this._state.running.push(space.key.toHex());
+    const spaceKey = space.key.toHex();
+    const currentState = this._registry.get(this._stateAtom);
+    this._registry.set(this._stateAtom, { running: [...currentState.running, spaceKey] });
     if (targetIndex > currentIndex) {
       const migrations = this.migrations.slice(currentIndex, targetIndex);
       for (const migration of migrations) {
         const builder = new MigrationBuilder(space);
         await migration.next({ space, builder });
-        builder.changeProperties((propertiesStructure) => {
+        await builder.changeProperties((propertiesStructure) => {
           invariant(this.versionProperty, 'Migrations namespace not set');
           propertiesStructure.data[this.versionProperty] = migration.version;
         });
         await builder._commit();
       }
     }
-    this._state.running.splice(this._state.running.indexOf(space.key.toHex()), 1);
+    const finalState = this._registry.get(this._stateAtom);
+    this._registry.set(this._stateAtom, { running: finalState.running.filter((key) => key !== spaceKey) });
 
     return true;
   }