@palantir/pack.state.core 0.0.1-beta.1

package/src/service/InMemoryDocumentService.ts

@@ -0,0 +1,162 @@
+/*
+ * Copyright 2025 Palantir Technologies, Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { ModuleConfigTuple, PackAppInternal } from "@palantir/pack.core";
+import { generateId } from "@palantir/pack.core";
+import type {
+  DocumentId,
+  DocumentMetadata,
+  DocumentRef,
+  DocumentSchema,
+} from "@palantir/pack.document-schema.model-types";
+import type * as Y from "yjs";
+import { createDocumentServiceConfig } from "../DocumentServiceModule.js";
+import { createDocRef } from "../types/DocumentRefImpl.js";
+import type { DocumentService } from "../types/DocumentService.js";
+import { DocumentLoadStatus } from "../types/DocumentService.js";
+import type { InternalYjsDoc } from "./BaseYjsDocumentService.js";
+import { BaseYjsDocumentService } from "./BaseYjsDocumentService.js";
+
+export interface InMemoryDocumentServiceOptions {
+  /**
+   * Automatically create documents when referenced
+   * @default true
+   */
+  readonly autoCreateDocuments?: boolean;
+}
+
+export function createInMemoryDocumentServiceConfig(
+  { autoCreateDocuments = true }: InMemoryDocumentServiceOptions = {},
+): ModuleConfigTuple<DocumentService> {
+  const config: InMemoryDocumentServiceOptions = {
+    autoCreateDocuments,
+  };
+
+  return createDocumentServiceConfig(internalCreateInMemoryDocumentService, config);
+}
+
+export function internalCreateInMemoryDocumentService(
+  app: PackAppInternal,
+  options: InMemoryDocumentServiceOptions,
+): BaseYjsDocumentService {
+  return new InMemoryDocumentService(app, options);
+}
+
+class InMemoryDocumentService extends BaseYjsDocumentService {
+  constructor(
+    app: PackAppInternal,
+    readonly config: InMemoryDocumentServiceOptions,
+  ) {
+    super(
+      app,
+      app.config.logger.child({}, { level: "debug", msgPrefix: "InMemoryDocumentService" }),
+    );
+  }
+
+  override createInternalDoc(
+    ref: DocumentRef,
+    metadata?: DocumentMetadata,
+    yDoc?: Y.Doc,
+  ): InternalYjsDoc {
+    return this.createBaseInternalDoc(ref, metadata, yDoc);
+  }
+
+  get hasMetadataSubscriptions() {
+    return Array.from(this.documents.values()).some(doc =>
+      this.hasSubscriptions(doc) && doc.metadataSubscribers.size > 0
+    );
+  }
+
+  get hasStateSubscriptions() {
+    return Array.from(this.documents.values()).some(doc =>
+      this.hasSubscriptions(doc) && doc.docStateSubscribers.size > 0
+    );
+  }
+
+  readonly createDocument = <T extends DocumentSchema>(
+    metadata: DocumentMetadata,
+    schema: T,
+  ): Promise<DocumentRef<T>> => {
+    const id = generateDocumentId();
+    const docRef = createDocRef(this.app, id, schema);
+
+    const yDoc = this.initializeYDoc(schema);
+    this.getCreateInternalDoc(docRef, metadata, yDoc);
+
+    return Promise.resolve(docRef);
+  };
+
+  // Lifecycle method implementations
+  protected onMetadataSubscriptionOpened(
+    internalDoc: InternalYjsDoc,
+    docRef: DocumentRef,
+  ): void {
+    this.updateMetadataStatus(internalDoc, docRef, {
+      load: DocumentLoadStatus.LOADING,
+    });
+
+    if (this.config.autoCreateDocuments === false && internalDoc.metadata == null) {
+      this.updateMetadataStatus(internalDoc, docRef, {
+        error: new Error("Document not found and autoCreateDocuments is disabled"),
+        load: DocumentLoadStatus.ERROR,
+      });
+      return;
+    }
+
+    this.updateMetadataStatus(internalDoc, docRef, {
+      load: DocumentLoadStatus.LOADED,
+    });
+  }
+
+  protected onDataSubscriptionOpened(
+    internalDoc: InternalYjsDoc,
+    docRef: DocumentRef,
+  ): void {
+    this.updateDataStatus(internalDoc, docRef, {
+      load: DocumentLoadStatus.LOADING,
+    });
+
+    if (this.config.autoCreateDocuments === false && internalDoc.metadata == null) {
+      this.updateDataStatus(internalDoc, docRef, {
+        error: new Error("Document not found and autoCreateDocuments is disabled"),
+        load: DocumentLoadStatus.ERROR,
+      });
+      return;
+    }
+
+    this.updateDataStatus(internalDoc, docRef, {
+      load: DocumentLoadStatus.LOADED,
+    });
+  }
+
+  protected onMetadataSubscriptionClosed(
+    _internalDoc: InternalYjsDoc,
+    _docRef: DocumentRef,
+  ): void {
+    // No cleanup needed for in-memory service
+  }
+
+  protected onDataSubscriptionClosed(
+    _internalDoc: InternalYjsDoc,
+    _docRef: DocumentRef,
+  ): void {
+    // No cleanup needed for in-memory service
+  }
+}
+
+function generateDocumentId(): DocumentId {
+  return generateId() as DocumentId;
+}

package/src/service/YjsSchemaMapper.ts

@@ -0,0 +1,194 @@
+/*
+ * Copyright 2025 Palantir Technologies, Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  type DocumentSchema,
+  getMetadata,
+  type Model,
+  type ModelData,
+  type RecordId,
+} from "@palantir/pack.document-schema.model-types";
+import * as Y from "yjs";
+
+export function initializeDocumentStructure(
+  yDoc: Y.Doc,
+  schema: DocumentSchema,
+): void {
+  // Initialize storage for each model type using storageName directly on the root Y.Doc
+  Object.values(schema).forEach(modelEntry => {
+    yDoc.getMap(getMetadata(modelEntry).name);
+  });
+}
+
+export function getRecordsMap(
+  yDoc: Y.Doc,
+  storageName: string,
+): Y.Map<unknown> {
+  return yDoc.getMap(storageName);
+}
+
+export function getRecordData(
+  yDoc: Y.Doc,
+  storageName: string,
+  recordId: RecordId,
+): Y.Map<unknown> | undefined {
+  const recordsCollection = getRecordsMap(yDoc, storageName);
+  return recordsCollection.get(recordId as string) as Y.Map<unknown> | undefined;
+}
+
+export function setRecord(
+  yDoc: Y.Doc,
+  storageName: string,
+  recordId: RecordId,
+  state: ModelData<Model>,
+): boolean {
+  const recordsCollection = getRecordsMap(yDoc, storageName);
+  const currentRecord = recordsCollection.get(recordId as string) as Y.Map<unknown> | undefined;
+  const wasExisting = currentRecord != null;
+
+  // Use transaction for atomic update
+  yDoc.transact(() => {
+    if (currentRecord != null) {
+      // Clear existing record completely
+      currentRecord.clear();
+      populateYMapFromState(currentRecord, state);
+    } else {
+      // Create new record
+      const newRecord = new Y.Map();
+      populateYMapFromState(newRecord, state);
+      recordsCollection.set(recordId as string, newRecord);
+    }
+  });
+
+  return wasExisting;
+}
+
+export function getRecordSnapshot(
+  yDoc: Y.Doc,
+  storageName: string,
+  recordId: RecordId,
+): unknown {
+  const data = getRecordData(yDoc, storageName, recordId);
+  if (!data) {
+    return undefined;
+  }
+
+  return yMapToState(data);
+}
+
+export function updateRecord(
+  yDoc: Y.Doc,
+  storageName: string,
+  recordId: RecordId,
+  partialState: Partial<ModelData<Model>>,
+): boolean {
+  const recordsCollection = getRecordsMap(yDoc, storageName);
+  const currentRecord = recordsCollection.get(recordId as string) as Y.Map<unknown> | undefined;
+
+  if (currentRecord == null) {
+    return false; // Record doesn't exist, cannot update
+  }
+
+  // Use transaction for atomic partial update
+  yDoc.transact(() => {
+    updateYMapFromPartialState(currentRecord, partialState);
+  });
+
+  return true;
+}
+
+export function getAllRecordIds(yDoc: Y.Doc, storageName: string): RecordId[] {
+  const recordsCollection = getRecordsMap(yDoc, storageName);
+  return Array.from(recordsCollection.keys()).map(key => key as RecordId);
+}
+
+function populateYMapFromState(
+  yMap: Y.Map<unknown>,
+  state: unknown,
+): void {
+  if (state != null && typeof state === "object") {
+    Object.entries(state).forEach(([key, value]) => {
+      if (value === undefined) return;
+
+      if (Array.isArray(value)) {
+        const yArray = new Y.Array();
+        value.forEach(item => {
+          yArray.push([item]);
+        });
+        yMap.set(key, yArray);
+      } else if (typeof value === "object" && value != null) {
+        const nestedMap = new Y.Map();
+        populateYMapFromState(nestedMap, value as Model);
+        yMap.set(key, nestedMap);
+      } else {
+        yMap.set(key, value);
+      }
+    });
+  }
+}
+
+function updateYMapFromPartialState(
+  yMap: Y.Map<unknown>,
+  partialState: Partial<unknown>,
+): void {
+  if (typeof partialState === "object") {
+    Object.entries(partialState).forEach(([key, value]) => {
+      if (value === undefined) {
+        // Remove the field if undefined is explicitly provided
+        yMap.delete(key);
+        return;
+      }
+
+      if (Array.isArray(value)) {
+        const yArray = new Y.Array();
+        value.forEach(item => {
+          yArray.push([item]);
+        });
+        yMap.set(key, yArray);
+      } else if (typeof value === "object" && value != null) {
+        // For nested objects, we need to handle partial updates
+        const existingValue = yMap.get(key);
+        if (existingValue instanceof Y.Map) {
+          // Recursively update nested Y.Map
+          updateYMapFromPartialState(existingValue, value as Partial<unknown>);
+        } else {
+          // Replace with new nested map if existing value is not a Y.Map
+          const nestedMap = new Y.Map();
+          populateYMapFromState(nestedMap, value as Model);
+          yMap.set(key, nestedMap);
+        }
+      } else {
+        yMap.set(key, value);
+      }
+    });
+  }
+}
+
+function yMapToState(yMap: Y.Map<unknown>): ModelData<Model> {
+  const state: Record<string, unknown> = {};
+
+  yMap.forEach((value, key) => {
+    if (value instanceof Y.Array) {
+      state[key] = value.toArray();
+    } else if (value instanceof Y.Map) {
+      state[key] = yMapToState(value);
+    } else {
+      state[key] = value;
+    }
+  });
+
+  return state as ModelData<Model>;
+}

package/src/types/DocumentRefImpl.ts

@@ -0,0 +1,98 @@
+/*
+ * Copyright 2025 Palantir Technologies, Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { PackAppInternal } from "@palantir/pack.core";
+import type {
+  DocumentId,
+  DocumentMetadata,
+  DocumentRef,
+  DocumentSchema,
+  DocumentState,
+  Model,
+  RecordCollectionRef,
+  Unsubscribe,
+} from "@palantir/pack.document-schema.model-types";
+import { DocumentRefBrand } from "@palantir/pack.document-schema.model-types";
+import type { StateModuleImpl } from "./StateModule.js";
+import { getStateModule } from "./StateModule.js";
+
+const INVALID_DOC_REF_ID: DocumentId = "INVALID_DOC_REF";
+const INVALID_DOC_REF: DocumentRef = Object.freeze(
+  {
+    id: INVALID_DOC_REF_ID,
+    schema: {} as DocumentSchema,
+    [DocumentRefBrand]: DocumentRefBrand,
+    getDocSnapshot: () => Promise.reject(new Error("Invalid document reference")),
+    getRecords: () => {
+      throw new Error("Invalid document reference");
+    },
+    onMetadataChange: () => () => {},
+    onStateChange: () => () => {},
+  } as const,
+);
+
+export const createDocRef = <const D extends DocumentSchema>(
+  app: PackAppInternal,
+  id: DocumentId,
+  schema: D,
+): DocumentRef<D> => {
+  return new DocumentRefImpl(app, id, schema);
+};
+
+export function invalidDocRef<D extends DocumentSchema = DocumentSchema>(): DocumentRef<D> {
+  return INVALID_DOC_REF as DocumentRef<D>;
+}
+
+export function isValidDocRef<D extends DocumentSchema = DocumentSchema>(
+  docRef: DocumentRef<D>,
+): docRef is DocumentRef<D> {
+  return docRef.id !== INVALID_DOC_REF_ID && docRef.id !== "";
+}
+
+class DocumentRefImpl<T extends DocumentSchema> implements DocumentRef<T> {
+  readonly id: DocumentId;
+  readonly schema: T;
+  declare readonly [DocumentRefBrand]: typeof DocumentRefBrand;
+  readonly #stateModule: StateModuleImpl;
+
+  constructor(app: PackAppInternal, id: DocumentId, schema: T) {
+    this.#stateModule = getStateModule(app) as StateModuleImpl;
+    this.id = id;
+    this.schema = schema;
+  }
+
+  async getDocSnapshot(): Promise<DocumentState<T>> {
+    return this.#stateModule.getDocumentSnapshot(this);
+  }
+
+  getRecords<R extends Model>(
+    model: R,
+  ): RecordCollectionRef<R> {
+    return this.#stateModule.getCreateRecordCollectionRef(this, model);
+  }
+
+  onMetadataChange(
+    cb: (docRef: DocumentRef<T>, metadata: DocumentMetadata) => void,
+  ): Unsubscribe {
+    return this.#stateModule.onMetadataChange(this, cb);
+  }
+
+  onStateChange(
+    callback: (docRef: DocumentRef<T>) => void,
+  ): Unsubscribe {
+    return this.#stateModule.onStateChange(this, callback);
+  }
+}

package/src/types/DocumentService.ts

@@ -0,0 +1,210 @@
+/*
+ * Copyright 2025 Palantir Technologies, Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { Unsubscribe } from "@palantir/pack.core";
+import type {
+  DocumentId,
+  DocumentMetadata,
+  DocumentRef,
+  DocumentSchema,
+  DocumentState,
+  Model,
+  ModelData,
+  RecordCollectionRef,
+  RecordId,
+  RecordRef,
+} from "@palantir/pack.document-schema.model-types";
+
+export const DocumentLoadStatus = {
+  UNLOADED: "unloaded", // Not yet loaded
+  LOADING: "loading", // Initial load in progress
+  LOADED: "loaded", // Successfully loaded
+  ERROR: "error", // Load failed
+} as const;
+export type DocumentLoadStatus = typeof DocumentLoadStatus[keyof typeof DocumentLoadStatus];
+
+export const DocumentLiveStatus = {
+  DISCONNECTED: "disconnected", // Not syncing
+  CONNECTING: "connecting", // Establishing connection
+  CONNECTED: "connected", // Live syncing active
+  ERROR: "error", // Connection error
+} as const;
+export type DocumentLiveStatus = typeof DocumentLiveStatus[keyof typeof DocumentLiveStatus];
+
+export type DocumentSyncStatus = {
+  readonly error?: unknown;
+  readonly live: DocumentLiveStatus;
+  readonly load: DocumentLoadStatus;
+};
+
+export type DocumentStatus = {
+  readonly metadata: DocumentSyncStatus;
+  readonly data: DocumentSyncStatus;
+  readonly metadataError?: unknown;
+  readonly dataError?: unknown;
+};
+
+export type DocumentStatusChangeCallback = (
+  docRef: DocumentRef,
+  status: DocumentStatus,
+) => void;
+
+export type DocumentMetadataChangeCallback<
+  T extends DocumentSchema = DocumentSchema,
+> = (docRef: DocumentRef<T>, metadata: DocumentMetadata) => void;
+
+export type DocumentStateChangeCallback<
+  T extends DocumentSchema = DocumentSchema,
+> = (docRef: DocumentRef<T>) => void;
+
+export type RecordCollectionChangeCallback<M extends Model = Model> = (
+  items: readonly RecordRef<M>[],
+) => void;
+
+export type RecordChangeCallback<M extends Model = Model> = (
+  snapshot: ModelData<M>,
+  record: RecordRef<M>,
+) => void;
+
+export type RecordDeleteCallback<M extends Model = Model> = (
+  record: RecordRef<M>,
+) => void;
+
+export interface DocumentService {
+  readonly hasMetadataSubscriptions: boolean;
+  readonly hasStateSubscriptions: boolean;
+
+  readonly createDocument: <T extends DocumentSchema>(
+    metadata: DocumentMetadata,
+    schema: T,
+  ) => Promise<DocumentRef<T>>;
+
+  readonly createDocRef: <const T extends DocumentSchema>(
+    id: DocumentId,
+    schema: T,
+  ) => DocumentRef<T>;
+
+  readonly getCreateRecordCollectionRef: <const M extends Model>(
+    docRef: DocumentRef,
+    model: M,
+  ) => RecordCollectionRef<M>;
+
+  readonly getCreateRecordRef: <const M extends Model>(
+    docRef: DocumentRef,
+    id: RecordId,
+    model: M,
+  ) => RecordRef<M>;
+
+  readonly getDocumentSnapshot: <T extends DocumentSchema>(
+    docRef: DocumentRef<T>,
+  ) => Promise<DocumentState<T>>;
+
+  readonly getRecordSnapshot: <R extends Model>(
+    record: RecordRef<R>,
+  ) => Promise<ModelData<R>>;
+
+  readonly setRecord: <R extends Model>(
+    record: RecordRef<R>,
+    state: ModelData<R>,
+  ) => Promise<void>;
+
+  readonly updateRecord: <R extends Model>(
+    record: RecordRef<R>,
+    partialState: Partial<ModelData<R>>,
+  ) => Promise<void>;
+
+  // Collection methods
+  readonly getRecord: <M extends Model>(
+    collection: RecordCollectionRef<M>,
+    id: RecordId,
+  ) => RecordRef<M> | undefined;
+
+  readonly hasRecord: <M extends Model>(
+    collection: RecordCollectionRef<M>,
+    id: RecordId,
+  ) => boolean;
+
+  readonly setCollectionRecord: <M extends Model>(
+    collection: RecordCollectionRef<M>,
+    id: RecordId,
+    state: ModelData<M>,
+  ) => Promise<void>;
+
+  readonly deleteRecord: <M extends Model>(
+    record: RecordRef<M>,
+  ) => Promise<void>;
+
+  readonly getCollectionSize: <M extends Model>(
+    collection: RecordCollectionRef<M>,
+  ) => number;
+
+  readonly getCollectionRecords: <M extends Model>(
+    collection: RecordCollectionRef<M>,
+  ) => RecordRef<M>[];
+
+  readonly onCollectionItemsAdded: <M extends Model>(
+    collection: RecordCollectionRef<M>,
+    callback: RecordCollectionChangeCallback<M>,
+  ) => Unsubscribe;
+
+  readonly onCollectionItemsChanged: <M extends Model>(
+    collection: RecordCollectionRef<M>,
+    callback: RecordCollectionChangeCallback<M>,
+  ) => Unsubscribe;
+
+  readonly onCollectionItemsDeleted: <M extends Model>(
+    collection: RecordCollectionRef<M>,
+    callback: RecordCollectionChangeCallback<M>,
+  ) => Unsubscribe;
+
+  readonly onMetadataChange: <T extends DocumentSchema>(
+    docRef: DocumentRef<T>,
+    callback: DocumentMetadataChangeCallback<T>,
+  ) => Unsubscribe;
+
+  readonly onStateChange: <T extends DocumentSchema>(
+    docRef: DocumentRef<T>,
+    callback: DocumentStateChangeCallback<T>,
+  ) => Unsubscribe;
+
+  readonly onRecordChanged: <M extends Model>(
+    record: RecordRef<M>,
+    callback: RecordChangeCallback<M>,
+  ) => Unsubscribe;
+
+  readonly onRecordDeleted: <M extends Model>(
+    record: RecordRef<M>,
+    callback: RecordDeleteCallback<M>,
+  ) => Unsubscribe;
+
+  // Status methods
+  readonly getDocumentStatus: <T extends DocumentSchema>(
+    docRef: DocumentRef<T>,
+  ) => DocumentStatus;
+
+  readonly onStatusChange: <T extends DocumentSchema>(
+    docRef: DocumentRef<T>,
+    callback: DocumentStatusChangeCallback,
+  ) => Unsubscribe;
+
+  readonly waitForMetadataLoad: <T extends DocumentSchema>(
+    docRef: DocumentRef<T>,
+  ) => Promise<void>;
+
+  readonly waitForDataLoad: <T extends DocumentSchema>(
+    docRef: DocumentRef<T>,
+  ) => Promise<void>;
+}

package/src/types/DocumentServiceConfig.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2025 Palantir Technologies, Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { PackAppInternal } from "@palantir/pack.core";
+import type { DocumentService } from "./DocumentService.js";
+
+export type WithDocumentServiceInit = {
+  readonly init: (app: PackAppInternal, config: unknown) => DocumentService;
+};