@instantdb/core 0.22.86-experimental.split-store.20178922132.1 → 0.22.87-experimental.drewh-explorer-component.20180358679.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@instantdb/core",
-  "version": "0.22.86-experimental.split-store.20178922132.1",
+  "version": "0.22.87-experimental.drewh-explorer-component.20180358679.1",
   "description": "Instant's core local abstraction",
   "homepage": "https://github.com/instantdb/instant/tree/main/client/packages/core",
   "repository": {
@@ -53,7 +53,7 @@
   "dependencies": {
     "mutative": "^1.0.10",
     "uuid": "^11.1.0",
-    "@instantdb/version": "0.22.86-experimental.split-store.20178922132.1"
+    "@instantdb/version": "0.22.87-experimental.drewh-explorer-component.20180358679.1"
   },
   "scripts": {
     "test": "vitest",

package/src/Reactor.js

@@ -1,8 +1,8 @@
 // @ts-check
 import weakHash from './utils/weakHash.ts';
 import instaql from './instaql.ts';
-import * as instaml from './instaml.ts';
-import * as sts from './store.ts';
+import * as instaml from './instaml.js';
+import * as s from './store.ts';
 import uuid from './utils/uuid.ts';
 import IndexedDBStorage from './IndexedDBStorage.ts';
 import WindowNetworkListener from './WindowNetworkListener.js';
@@ -36,8 +36,6 @@ import { SyncTable } from './SyncTable.ts';
 /** @typedef {import('./Connection.ts').Connection} Connection */
 /** @typedef {import('./Connection.ts').TransportType} TransportType */
 /** @typedef {import('./Connection.ts').EventSourceConstructor} EventSourceConstructor */
-/** @typedef {import('./reactorTypes.ts').QuerySub} QuerySub */
-/** @typedef {import('./reactorTypes.ts').QuerySubInStorage} QuerySubInStorage */
 
 const STATUS = {
   CONNECTING: 'connecting',
@@ -109,52 +107,27 @@ const ignoreLogging = {
   'patch-presence': true,
 };
 
-/**
- * @param {QuerySubInStorage} x
- * @param {boolean | null} useDateObjects
- * @returns {QuerySub}
- */
 function querySubFromStorage(x, useDateObjects) {
   const v = typeof x === 'string' ? JSON.parse(x) : x;
 
   if (v?.result?.store) {
-    const attrsStore = sts.attrsStoreFromJSON(
-      v.result.attrsStore,
-      v.result.store,
-    );
-    if (attrsStore) {
-      const storeJSON = v.result.store;
-      v.result.store = sts.fromJSON(attrsStore, {
-        ...storeJSON,
-        useDateObjects: useDateObjects,
-      });
-      v.result.attrsStore = attrsStore;
-    }
+    const storeJSON = v.result.store;
+    v.result.store = s.fromJSON({
+      ...storeJSON,
+      useDateObjects: useDateObjects,
+    });
   }
 
   return v;
 }
 
-/**
- *
- * @param {string} _key
- * @param {QuerySub} sub
- * @returns QuerySubInStorage
- */
 function querySubToStorage(_key, sub) {
-  const { result, ...rest } = sub;
-  const jsonSub = /** @type {import('./reactorTypes.ts').QuerySubInStorage} */ (
-    rest
-  );
-  if (result) {
-    /** @type {import('./reactorTypes.ts').QuerySubResultInStorage} */
-    const jsonResult = {
-      ...result,
-      store: sts.toJSON(result.store),
-      attrsStore: result.attrsStore.toJSON(),
+  const jsonSub = { ...sub };
+  if (sub.result?.store) {
+    jsonSub.result = {
+      ...sub.result,
+      store: s.toJSON(sub.result.store),
     };
-
-    jsonSub.result = jsonResult;
   }
   return jsonSub;
 }
@@ -203,13 +176,12 @@ function sortedMutationEntries(entries) {
  * @template {import('./presence.ts').RoomSchemaShape} [RoomSchema = {}]
  */
 export default class Reactor {
-  /** @type {sts.AttrsStore | undefined} */
   attrs;
   _isOnline = true;
   _isShutdown = false;
   status = STATUS.CONNECTING;
 
-  /** @type {PersistedObject<string, QuerySub, QuerySubInStorage>} */
+  /** @type {PersistedObject} */
   querySubs;
 
   /** @type {PersistedObject} */
@@ -334,15 +306,14 @@ export default class Reactor {
         useDateObjects: this.config.useDateObjects,
       },
       this._log,
-      (triples) => {
-        return sts.createStore(
-          this.ensureAttrs(),
+      (triples) =>
+        s.createStore(
+          this.attrs,
           triples,
           this.config.enableCardinalityInference,
+          this._linkIndex,
           this.config.useDateObjects,
-        );
-      },
-      () => this.ensureAttrs(),
+        ),
     );
 
     this._oauthCallbackResponse = this._oauthLoginInit();
@@ -382,13 +353,6 @@ export default class Reactor {
     }
   }
 
-  ensureAttrs() {
-    if (!this.attrs) {
-      throw new Error('attrs have not loaded.');
-    }
-    return this.attrs;
-  }
-
   updateSchema(schema) {
     this.config = {
       ...this.config,
@@ -419,7 +383,7 @@ export default class Reactor {
       serialize: querySubToStorage,
       parse: (_key, x) => querySubFromStorage(x, this.config.useDateObjects),
       // objectSize
-      objectSize: (x) => x?.result?.store?.triples?.length ?? 0,
+      objectSize: (x) => x.result?.store?.triples?.length ?? 0,
       logger: this._log,
       preloadEntryCount: 10,
       gc: {
@@ -588,13 +552,13 @@ export default class Reactor {
         const pageInfo = result?.[0]?.data?.['page-info'];
         const aggregate = result?.[0]?.data?.['aggregate'];
         const triples = extractTriples(result);
-        const store = sts.createStore(
-          this.ensureAttrs(),
+        const store = s.createStore(
+          this.attrs,
           triples,
           enableCardinalityInference,
+          this._linkIndex,
           this.config.useDateObjects,
         );
-
         this.querySubs.updateInPlace((prev) => {
           if (!prev[hash]) {
             this._log.info('Missing value in querySubs', { hash, q });
@@ -602,7 +566,6 @@ export default class Reactor {
           }
           prev[hash].result = {
             store,
-            attrsStore: this.ensureAttrs(),
             pageInfo,
             aggregate,
             processedTxId: msg['processed-tx-id'],
@@ -640,7 +603,7 @@ export default class Reactor {
         this._cleanupPendingMutationsTimeout();
 
         const rewrittenMutations = this._rewriteMutations(
-          this.ensureAttrs(),
+          this.attrs,
           this._pendingMutations(),
           processedTxId,
         );
@@ -661,15 +624,15 @@ export default class Reactor {
           const result = x['instaql-result'];
           const hash = weakHash(q);
           const triples = extractTriples(result);
-          const store = sts.createStore(
-            this.ensureAttrs(),
+          const store = s.createStore(
+            this.attrs,
             triples,
             enableCardinalityInference,
+            this._linkIndex,
             this.config.useDateObjects,
           );
           const newStore = this._applyOptimisticUpdates(
             store,
-            this.ensureAttrs(),
             mutations,
             processedTxId,
           );
@@ -684,13 +647,7 @@ export default class Reactor {
               this._log.error('Missing value in querySubs', { hash, q });
               return;
             }
-            prev[hash].result = {
-              store,
-              attrsStore: this.ensureAttrs(),
-              pageInfo,
-              aggregate,
-              processedTxId,
-            };
+            prev[hash].result = { store, pageInfo, aggregate, processedTxId };
           });
         });
 
@@ -707,7 +664,7 @@ export default class Reactor {
         this._inFlightMutationEventIds.delete(eventId);
 
         const muts = this._rewriteMutations(
-          this.ensureAttrs(),
+          this.attrs,
           this._pendingMutations(),
         );
         const prevMutation = muts.get(eventId);
@@ -724,18 +681,12 @@ export default class Reactor {
           });
         });
 
-        const newAttrs = [];
-        for (const step of prevMutation['tx-steps']) {
-          if (step[0] === 'add-attr') {
-            const attr = step[1];
-            newAttrs.push(attr);
-          }
-        }
-        if (newAttrs.length) {
-          const existingAttrs = Object.values(this.ensureAttrs());
-          this._setAttrs([...existingAttrs, ...newAttrs]);
-          this._setAttrs(newAttrs);
-        }
+        const newAttrs = prevMutation['tx-steps']
+          .filter(([action, ..._args]) => action === 'add-attr')
+          .map(([_action, attr]) => attr)
+          .concat(Object.values(this.attrs));
+
+        this._setAttrs(newAttrs);
 
         this._finishTransaction('synced', eventId);
 
@@ -919,13 +870,10 @@ export default class Reactor {
   }
 
   _setAttrs(attrs) {
-    this.attrs = new sts.AttrsStore(
-      attrs.reduce((acc, attr) => {
-        acc[attr.id] = attr;
-        return acc;
-      }, {}),
-      this._linkIndex,
-    );
+    this.attrs = attrs.reduce((acc, attr) => {
+      acc[attr.id] = attr;
+      return acc;
+    }, {});
 
     this.notifyAttrsSubs();
   }
@@ -1074,23 +1022,17 @@ export default class Reactor {
   // We remove `add-attr` commands for attrs that already exist.
   // We update `add-triple` and `retract-triple` commands to use the
   // server attr-ids.
-  /**
-   *
-   * @param {sts.AttrsStore} attrs
-   * @param {any} muts
-   * @param {number} [processedTxId]
-   */
   _rewriteMutations(attrs, muts, processedTxId) {
     if (!attrs) return muts;
     if (!muts) return new Map();
     const findExistingAttr = (attr) => {
       const [_, etype, label] = attr['forward-identity'];
-      const existing = sts.getAttrByFwdIdentName(attrs, etype, label);
+      const existing = instaml.getAttrByFwdIdentName(attrs, etype, label);
       return existing;
     };
     const findReverseAttr = (attr) => {
       const [_, etype, label] = attr['forward-identity'];
-      const revAttr = sts.getAttrByReverseIdentName(attrs, etype, label);
+      const revAttr = instaml.getAttrByReverseIdentName(attrs, etype, label);
       return revAttr;
     };
     const mapping = { attrIdMap: {}, refSwapAttrIds: new Set() };
@@ -1167,9 +1109,6 @@ export default class Reactor {
   // ---------------------------
   // Transact
 
-  /**
-   * @returns {sts.AttrsStore}
-   */
   optimisticAttrs() {
     const pendingMutationSteps = [...this._pendingMutations().values()] // hack due to Map()
       .flatMap((x) => x['tx-steps']);
@@ -1187,26 +1126,23 @@ export default class Reactor {
       } else if (
         _action === 'update-attr' &&
         attr.id &&
-        this.attrs?.getAttr(attr.id)
+        this.attrs?.[attr.id]
       ) {
-        const fullAttr = { ...this.attrs.getAttr(attr.id), ...attr };
+        const fullAttr = { ...this.attrs[attr.id], ...attr };
         pendingAttrs.push(fullAttr);
       }
     }
 
-    if (!deletedAttrIds.size && !pendingAttrs.length) {
-      return this.attrs || new sts.AttrsStore({}, this._linkIndex);
-    }
+    const attrsWithoutDeleted = [
+      ...Object.values(this.attrs || {}),
+      ...pendingAttrs,
+    ].filter((a) => !deletedAttrIds.has(a.id));
 
-    const attrs = { ...(this.attrs?.attrs || {}) };
-    for (const attr of pendingAttrs) {
-      attrs[attr.id] = attr;
-    }
-    for (const id of deletedAttrIds) {
-      delete attrs[id];
-    }
+    const attrsRecord = Object.fromEntries(
+      attrsWithoutDeleted.map((a) => [a.id, a]),
+    );
 
-    return new sts.AttrsStore(attrs, this._linkIndex);
+    return attrsRecord;
   }
 
   /** Runs instaql on a query and a store */
@@ -1234,29 +1170,25 @@ export default class Reactor {
       return cached;
     }
 
-    const { store, attrsStore, pageInfo, aggregate, processedTxId } = result;
+    const { store, pageInfo, aggregate, processedTxId } = result;
     const mutations = this._rewriteMutationsSorted(
-      attrsStore,
+      store.attrs,
       pendingMutations,
     );
     const newStore = this._applyOptimisticUpdates(
       store,
-      attrsStore,
       mutations,
       processedTxId,
     );
-    const resp = instaql(
-      { store: newStore, attrsStore, pageInfo, aggregate },
-      q,
-    );
+    const resp = instaql({ store: newStore, pageInfo, aggregate }, q);
 
     return { data: resp, querySubVersion, pendingMutationsVersion };
   }
 
-  _applyOptimisticUpdates(store, attrsStore, mutations, processedTxId) {
+  _applyOptimisticUpdates(store, mutations, processedTxId) {
     for (const [_, mut] of mutations) {
       if (!mut['tx-id'] || (processedTxId && mut['tx-id'] > processedTxId)) {
-        store = sts.transact(store, attrsStore, mut['tx-steps']);
+        store = s.transact(store, mut['tx-steps']);
       }
     }
     return store;
@@ -1316,7 +1248,7 @@ export default class Reactor {
     try {
       const txSteps = instaml.transform(
         {
-          attrsStore: this.optimisticAttrs(),
+          attrs: this.optimisticAttrs(),
           schema: this.config.schema,
           stores: Object.values(this.querySubs.currentValue).map(
             (sub) => sub?.result?.store,
@@ -1434,7 +1366,7 @@ export default class Reactor {
       });
 
     const muts = this._rewriteMutationsSorted(
-      this.ensureAttrs(),
+      this.attrs,
       this._pendingMutations(),
     );
     muts.forEach(([eventId, mut]) => {
@@ -1890,7 +1822,7 @@ export default class Reactor {
     this.attrsCbs.push(cb);
 
     if (this.attrs) {
-      cb(this.attrs.attrs);
+      cb(this.attrs);
     }
 
     return () => {

package/src/SyncTable.ts

@@ -16,12 +16,12 @@ type SubState = {
 
 type SubEntity = {
   entity: any;
-  store: s.Store;
+  store: any;
   serverCreatedAt: number;
 };
 
 type SubValues = {
-  attrsStore: s.AttrsStore;
+  attrs: Record<string, any>;
   entities: Array<SubEntity>;
 };
 
@@ -38,21 +38,8 @@ type Sub = {
   updatedAt: number;
 };
 
-type SubEntityInStorage = {
-  entity: any;
-  store: s.StoreJson;
-  serverCreatedAt: number;
-};
-
-type SubValuesInStorage = {
-  attrsStore: s.AttrsStoreJson;
-  entities: Array<SubEntityInStorage>;
-};
-
 // We could make a better type for this if we had a return type for s.toJSON
-type SubInStorage = Omit<Sub, 'values'> & {
-  values: SubValuesInStorage;
-};
+type SubInStorage = Sub;
 
 type StartMsg = {
   op: 'start-sync';
@@ -109,38 +96,36 @@ type Config = { useDateObjects: boolean };
 function syncSubFromStorage(sub: SubInStorage, useDateObjects: boolean): Sub {
   const values = sub.values;
   if (values) {
-    const attrsStore = s.attrsStoreFromJSON(values.attrsStore, null);
-    if (attrsStore) {
-      for (const e of values.entities || []) {
-        e.store.useDateObjects = useDateObjects;
-        (e as unknown as SubEntity).store = s.fromJSON(attrsStore, e.store);
-      }
-      (values as unknown as SubValues).attrsStore = attrsStore;
+    for (const e of values.entities || []) {
+      e.store.useDateObjects = useDateObjects;
+      e.store.attrs = values.attrs;
+      e.store = s.fromJSON(e.store);
     }
   }
 
-  return sub as unknown as Sub;
+  return sub;
 }
 
 function syncSubToStorage(_k: string, sub: Sub): SubInStorage {
-  if (sub.values) {
-    const entities: SubEntityInStorage[] = [];
+  if (sub.values?.entities) {
+    const entities: SubEntity[] = [];
     for (const e of sub.values?.entities) {
       const store = s.toJSON(e.store);
+      // We'll store the attrs once on values, and put the
+      // attrs back into the store on hydration
+      // @ts-ignore: ts doesn't want us to delete a non-optional
+      delete store['attrs'];
       entities.push({ ...e, store });
     }
-    return {
-      ...sub,
-      values: { attrsStore: sub.values.attrsStore.toJSON(), entities },
-    };
+    return { ...sub, values: { ...sub.values, entities } };
   } else {
-    return sub as unknown as SubInStorage;
+    return sub;
   }
 }
 
 function onMergeSub(
   _key: string,
-  storageSub: Sub,
+  storageSub: SubInStorage,
   inMemorySub: Sub | null,
 ): Sub {
   const storageTxId = storageSub?.state?.txId;
@@ -157,21 +142,13 @@ function onMergeSub(
   return storageSub || inMemorySub;
 }
 
-function queryEntity(sub: Sub, store: s.Store, attrsStore: s.AttrsStore) {
-  const res = instaql(
-    { store, attrsStore, pageInfo: null, aggregate: null },
-    sub.query,
-  );
+function queryEntity(sub: Sub, store: any) {
+  const res = instaql({ store, pageInfo: null, aggregate: null }, sub.query);
   return res.data[sub.table][0];
 }
 
-function getServerCreatedAt(
-  sub: Sub,
-  store: s.Store,
-  attrsStore: s.AttrsStore,
-  entityId: string,
-): number {
-  const aid = s.getAttrByFwdIdentName(attrsStore, sub.table, 'id')?.id;
+function getServerCreatedAt(sub: Sub, store: any, entityId: string): number {
+  const aid = s.getAttrByFwdIdentName(store, sub.table, 'id')?.id;
   if (!aid) {
     return -1;
   }
@@ -183,25 +160,23 @@ function getServerCreatedAt(
 }
 
 function applyChangesToStore(
-  store: s.Store,
-  attrsStore: s.AttrsStore,
+  store: any,
   changes: SyncUpdateTriplesMsg['txes'][number]['changes'],
 ): void {
   for (const { action, triple } of changes) {
     switch (action) {
       case 'added':
-        s.addTriple(store, attrsStore, triple);
+        s.addTriple(store, triple);
         break;
       case 'removed':
-        s.retractTriple(store, attrsStore, triple);
+        s.retractTriple(store, triple);
         break;
     }
   }
 }
 
 function changedFieldsOfChanges(
-  store: s.Store,
-  attrsStore: s.AttrsStore,
+  store: any,
   changes: SyncUpdateTriplesMsg['txes'][number]['changes'],
 ): {
   [eid: string]: SyncTransaction<
@@ -215,7 +190,7 @@ function changedFieldsOfChanges(
   const changedFields = {};
   for (const { action, triple } of changes) {
     const [e, a, v] = triple;
-    const field = attrsStore.getAttr(a)?.['forward-identity']?.[2];
+    const field = store.attrs[a]?.['forward-identity']?.[2];
     if (!field) continue;
 
     const fields = changedFields[e] ?? {};
@@ -250,19 +225,17 @@ function subData(sub: Sub, entities: NonNullable<Sub['values']>['entities']) {
   return { [sub.table]: entities.map((e) => e.entity) };
 }
 
-type CreateStore = (triples: Triple[]) => s.Store;
-
 // Updates the sub order field type if it hasn't been set
 // and returns the type. We have to wait until the attrs
 // are loaded before we can determine the type.
-function orderFieldTypeMutative(sub: Sub, getAttrs: () => s.AttrsStore) {
+function orderFieldTypeMutative(sub: Sub, createStore) {
   if (sub.orderFieldType) {
     return sub.orderFieldType;
   }
   const orderFieldType =
     sub.orderField === 'serverCreatedAt'
       ? 'number'
-      : s.getAttrByFwdIdentName(getAttrs(), sub.table, sub.orderField)?.[
+      : s.getAttrByFwdIdentName(createStore([]), sub.table, sub.orderField)?.[
           'checked-data-type'
         ];
 
@@ -441,22 +414,19 @@ export class SyncTable {
   private config: Config;
   private idToHash: { [subscriptionId: string]: string } = {};
   private log: Logger;
-  private createStore: CreateStore;
-  private getAttrs: () => s.AttrsStore;
+  private createStore: (triples: Triple[]) => any;
 
   constructor(
     trySend: TrySend,
     storage: StorageInterface,
     config: Config,
     log: Logger,
-    createStore: CreateStore,
-    getAttrs: () => s.AttrsStore,
+    createStore: (triples: Triple[]) => any,
   ) {
     this.trySend = trySend;
     this.config = config;
     this.log = log;
     this.createStore = createStore;
-    this.getAttrs = getAttrs;
 
     this.subs = new PersistedObject<string, Sub, SubInStorage>({
       persister: storage,
@@ -654,23 +624,19 @@ export class SyncTable {
 
     const values: SubValues = sub.values ?? {
       entities: [],
-      attrsStore: this.getAttrs(),
+      attrs: this.createStore([]).attrs,
     };
     sub.values = values;
     const entities = values.entities;
 
     for (const entRows of joinRows) {
       const store = this.createStore(entRows);
-      const entity = queryEntity(sub, store, values.attrsStore);
+      values.attrs = store.attrs;
+      const entity = queryEntity(sub, store);
       entities.push({
         store,
         entity,
-        serverCreatedAt: getServerCreatedAt(
-          sub,
-          store,
-          values.attrsStore,
-          entity.id,
-        ),
+        serverCreatedAt: getServerCreatedAt(sub, store, entity.id),
       });
       batch.push(entity);
     }
@@ -761,7 +727,7 @@ export class SyncTable {
 
       const values: SubValues = sub.values ?? {
         entities: [],
-        attrsStore: this.getAttrs(),
+        attrs: this.createStore([]).attrs,
       };
       const entities = values.entities;
       sub.values = values;
@@ -772,13 +738,11 @@ export class SyncTable {
         for (let i = 0; i < entities.length; i++) {
           const ent = entities[i];
           if (s.hasEntity(ent.store, eid)) {
-            applyChangesToStore(ent.store, values.attrsStore, changes);
-            const entity = queryEntity(sub, ent.store, values.attrsStore);
-            const changedFields = changedFieldsOfChanges(
-              ent.store,
-              values.attrsStore,
-              changes,
-            )[eid];
+            applyChangesToStore(ent.store, changes);
+            const entity = queryEntity(sub, ent.store);
+            const changedFields = changedFieldsOfChanges(ent.store, changes)[
+              eid
+            ];
             if (entity) {
               updated.push({
                 oldEntity: ent.entity,
@@ -799,8 +763,9 @@ export class SyncTable {
       // If we have anything left in byEid, then this must be a new entity we don't know about
       for (const [_eid, changes] of Object.entries(byEid)) {
         const store = this.createStore([]);
-        applyChangesToStore(store, values.attrsStore, changes);
-        const entity = queryEntity(sub, store, values.attrsStore);
+        values.attrs = store.attrs;
+        applyChangesToStore(store, changes);
+        const entity = queryEntity(sub, store);
         if (!entity) {
           this.log.error('No entity found after applying change', {
             sub,
@@ -812,12 +777,7 @@ export class SyncTable {
         entities.push({
           store,
           entity,
-          serverCreatedAt: getServerCreatedAt(
-            sub,
-            store,
-            values.attrsStore,
-            entity.id,
-          ),
+          serverCreatedAt: getServerCreatedAt(sub, store, entity.id),
         });
         added.push(entity);
       }
@@ -829,7 +789,7 @@ export class SyncTable {
         entities.splice(idx, 1);
       }
 
-      const orderFieldType = orderFieldTypeMutative(sub, this.getAttrs);
+      const orderFieldType = orderFieldTypeMutative(sub, this.createStore);
 
       sortEntitiesInPlace(sub, orderFieldType!, entities);
       this.notifyCbs(hash, {