@milaboratories/pl-middle-layer 1.10.12

package/src/pool/data.ts

@@ -0,0 +1,239 @@
+import { PObjectId, PObjectSpec } from '@platforma-sdk/model';
+import { PFrameInternal } from '@milaboratories/pl-model-middle-layer';
+import { PlTreeNodeAccessor, ResourceInfo } from '@milaboratories/pl-tree';
+import { assertNever } from '@milaboratories/ts-helpers';
+import { createHash } from 'crypto';
+import canonicalize from 'canonicalize';
+import {
+  isNullResourceId,
+  resourceType,
+  resourceTypeToString,
+  resourceTypesEqual
+} from '@milaboratories/pl-client';
+import { Writable } from 'utility-types';
+
+export function* allBlobs<B>(data: PFrameInternal.DataInfo<B>): Generator<B> {
+  switch (data.type) {
+    case 'Json':
+      return;
+    case 'JsonPartitioned':
+      for (const [, blob] of Object.entries(data.parts)) yield blob;
+      return;
+    case 'BinaryPartitioned':
+      for (const [, { index, values }] of Object.entries(data.parts)) {
+        yield index;
+        yield values;
+      }
+      return;
+    default:
+      assertNever(data);
+  }
+}
+
+function mapValues<T extends object, TResult>(
+  obj: T,
+  callback: (v: T[keyof T], key: keyof T) => TResult
+): { [P in keyof T]: TResult } {
+  return Object.fromEntries(
+    Object.entries(obj).map(([key, value]) => [key, callback(value, key as any)])
+  ) as any;
+}
+
+export function mapBlobs<B1, B2>(
+  data: PFrameInternal.DataInfo<B1>,
+  mapping: (blob: B1) => B2
+): PFrameInternal.DataInfo<B2> {
+  switch (data.type) {
+    case 'Json':
+      return { ...data };
+    case 'JsonPartitioned':
+      return { ...data, parts: mapValues(data.parts, mapping) };
+    case 'BinaryPartitioned':
+      return {
+        ...data,
+        parts: mapValues(data.parts, (v) => ({
+          index: mapping(v.index),
+          values: mapping(v.values)
+        }))
+      };
+    default:
+      assertNever(data);
+  }
+}
+
+export const PColumnDataJsonPartitioned = resourceType('PColumnData/JsonPartitioned', '1');
+export const PColumnDataJsonSuperPartitioned = resourceType(
+  'PColumnData/Partitioned/JsonPartitioned',
+  '1'
+);
+export const PColumnDataBinaryPartitioned = resourceType('PColumnData/BinaryPartitioned', '1');
+export const PColumnDataBinarySuperPartitioned = resourceType(
+  'PColumnData/Partitioned/BinaryPartitioned',
+  '1'
+);
+export const PColumnDataJson = resourceType('PColumnData/Json', '1');
+
+export type PColumnDataJsonResourceValue = {
+  keyLength: number;
+  data: Record<string, PFrameInternal.JsonDataValue>;
+};
+
+export type PColumnDataPartitionedResourceValue = {
+  partitionKeyLength: number;
+};
+
+export type PColumnDataSuperPartitionedResourceValue = {
+  superPartitionKeyLength: number;
+  partitionKeyLength: number;
+};
+
+export function parseDataInfoResource(
+  data: PlTreeNodeAccessor
+): PFrameInternal.DataInfo<ResourceInfo> {
+  if (!data.getIsReadyOrError()) throw new Error('Data not ready.');
+
+  const resourceData = data.getDataAsJson();
+  if (resourceData === undefined)
+    throw new Error('unexpected data info structure, no resource data');
+
+  if (resourceTypesEqual(data.resourceType, PColumnDataJson)) {
+    const dataContent = resourceData as PColumnDataJsonResourceValue;
+
+    return {
+      type: 'Json',
+      keyLength: dataContent.keyLength,
+      data: dataContent.data
+    };
+  } else if (resourceTypesEqual(data.resourceType, PColumnDataJsonPartitioned)) {
+    const meta = resourceData as PColumnDataPartitionedResourceValue;
+
+    const parts = Object.fromEntries(
+      data
+        .listInputFields()
+        .map((field) => [field, data.traverse({ field, errorIfFieldNotSet: true }).resourceInfo])
+    );
+
+    return {
+      type: 'JsonPartitioned',
+      partitionKeyLength: meta.partitionKeyLength,
+      parts
+    };
+  } else if (resourceTypesEqual(data.resourceType, PColumnDataJsonSuperPartitioned)) {
+    const meta = resourceData as PColumnDataSuperPartitionedResourceValue;
+
+    const parts: Record<string, ResourceInfo> = {};
+    for (const superKey of data.listInputFields()) {
+      const superPart = data.traverse({ field: superKey, errorIfFieldNotSet: true });
+      const keys = superPart.listInputFields();
+      if (keys === undefined) throw new Error(`no partition keys for super key ${superKey}`);
+
+      for (const key of keys) {
+        const partKey = superKey.slice(0, superKey.length - 1) + ',' + key.slice(1, key.length);
+        parts[partKey] = superPart.traverse({ field: key, errorIfFieldNotSet: true }).resourceInfo;
+      }
+    }
+
+    return {
+      type: 'JsonPartitioned',
+      partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,
+      parts
+    };
+  } else if (resourceTypesEqual(data.resourceType, PColumnDataBinaryPartitioned)) {
+    const meta = resourceData as PColumnDataPartitionedResourceValue;
+
+    const parts: Record<
+      string,
+      Partial<Writable<PFrameInternal.BinaryChunkInfo<ResourceInfo>>>
+    > = {};
+
+    // parsing the structure
+    for (const field of data.listInputFields()) {
+      if (field.endsWith('.index')) {
+        const partKey = field.slice(0, field.length - 6);
+        let part = parts[partKey];
+        if (part === undefined) {
+          part = {};
+          parts[partKey] = part;
+        }
+        part.index = data.traverse({ field, errorIfFieldNotSet: true }).resourceInfo;
+      } else if (field.endsWith('.values')) {
+        const partKey = field.slice(0, field.length - 7);
+        let part = parts[partKey];
+        if (part === undefined) {
+          part = {};
+          parts[partKey] = part;
+        }
+        part.values = data.traverse({ field, errorIfFieldNotSet: true }).resourceInfo;
+      } else throw new Error(`unrecognized part field name: ${field}`);
+    }
+
+    // structure validation
+    for (const [key, part] of Object.entries(parts)) {
+      if (part.index === undefined) throw new Error(`no index for part ${key}`);
+      if (part.values === undefined) throw new Error(`no values for part ${key}`);
+    }
+
+    return {
+      type: 'BinaryPartitioned',
+      partitionKeyLength: meta.partitionKeyLength,
+      parts: parts as Record<string, PFrameInternal.BinaryChunkInfo<ResourceInfo>>
+    };
+  } else if (resourceTypesEqual(data.resourceType, PColumnDataBinarySuperPartitioned)) {
+    const meta = resourceData as PColumnDataSuperPartitionedResourceValue;
+
+    const parts: Record<
+      string,
+      Partial<Writable<PFrameInternal.BinaryChunkInfo<ResourceInfo>>>
+    > = {};
+    for (const superKey of data.listInputFields()) {
+      const superData = data.traverse({ field: superKey, errorIfFieldNotSet: true });
+      const keys = superData.listInputFields();
+      if (keys === undefined) throw new Error(`no partition keys for super key ${superKey}`);
+
+      for (const field of keys) {
+        if (field.endsWith('.index')) {
+          const key = field.slice(0, field.length - 6);
+
+          const partKey = superKey.slice(0, superKey.length - 1) + ',' + key.slice(1, key.length);
+          let part = parts[partKey];
+          if (part === undefined) {
+            part = {};
+            parts[partKey] = part;
+          }
+          parts[partKey].index = superData.traverse({
+            field,
+            errorIfFieldNotSet: true
+          }).resourceInfo;
+        } else if (field.endsWith('.values')) {
+          const key = field.slice(0, field.length - 7);
+
+          const partKey = superKey.slice(0, superKey.length - 1) + ',' + key.slice(1, key.length);
+          let part = parts[partKey];
+          if (part === undefined) {
+            part = {};
+            parts[partKey] = part;
+          }
+          parts[partKey].values = superData.traverse({
+            field,
+            errorIfFieldNotSet: true
+          }).resourceInfo;
+        } else throw new Error(`unrecognized part field name: ${field}`);
+      }
+    }
+
+    return {
+      type: 'BinaryPartitioned',
+      partitionKeyLength: meta.superPartitionKeyLength + meta.partitionKeyLength,
+      parts: parts as Record<string, PFrameInternal.BinaryChunkInfo<ResourceInfo>>
+    };
+  }
+
+  throw new Error(`unsupported resource type: ${resourceTypeToString(data.resourceType)}`);
+}
+
+export function derivePObjectId(spec: PObjectSpec, data: PlTreeNodeAccessor): PObjectId {
+  const hash = createHash('sha256');
+  hash.update(canonicalize(spec)!);
+  hash.update(String(!isNullResourceId(data.originalId) ? data.originalId : data.id));
+  return hash.digest().toString('hex') as PObjectId;
+}

package/src/pool/driver.ts

@@ -0,0 +1,325 @@
+import { DownloadDriver } from '@milaboratories/pl-drivers';
+import type PFramesType from '@milaboratories/pframes-node';
+import { PFrameInternal } from '@milaboratories/pl-model-middle-layer';
+import { PlTreeNodeAccessor, ResourceInfo } from '@milaboratories/pl-tree';
+import { ComputableCtx, ComputableStableDefined } from '@milaboratories/computable';
+import {
+  CalculateTableDataRequest,
+  CalculateTableDataResponse,
+  FindColumnsRequest,
+  FindColumnsResponse,
+  LocalBlobHandleAndSize,
+  PColumnIdAndSpec,
+  PColumnSpec,
+  PFrameHandle,
+  PObjectId,
+  PTableColumnSpec,
+  PTableHandle,
+  PTableShape,
+  PTableVector,
+  TableRange,
+  UniqueValuesRequest,
+  UniqueValuesResponse,
+  PFrameDriver as SdkPFrameDriver,
+  PColumn,
+  mapPObjectData,
+  PFrameDef,
+  JoinEntry,
+  PTableDef,
+  mapPTableDef
+} from '@platforma-sdk/model';
+import { RefCountResourcePool } from './ref_count_pool';
+import { allBlobs, mapBlobs, parseDataInfoResource } from './data';
+import { createHash } from 'crypto';
+import { assertNever } from '@milaboratories/ts-helpers';
+import canonicalize from 'canonicalize';
+
+// special way of importing native node module
+const PFrames: PFramesType = require('@milaboratories/pframes-node');
+
+function blobKey(res: ResourceInfo): string {
+  return String(res.id);
+}
+
+type InternalPFrameData = PFrameDef<PFrameInternal.DataInfo<ResourceInfo>>;
+
+class PFrameHolder implements PFrameInternal.PFrameDataSource, Disposable {
+  public readonly pFrame: PFrameInternal.PFrame = new PFrames.PFrame();
+  private readonly blobIdToResource = new Map<string, ResourceInfo>();
+  private readonly blobHandleComputables = new Map<
+    string,
+    ComputableStableDefined<LocalBlobHandleAndSize>
+  >();
+
+  constructor(
+    private readonly blobDriver: DownloadDriver,
+    private readonly columns: InternalPFrameData
+  ) {
+    // pframe initialization
+    this.pFrame.setDataSource(this);
+    for (const column of columns) {
+      for (const blob of allBlobs(column.data)) this.blobIdToResource.set(blobKey(blob), blob);
+      const dataInfo = mapBlobs(column.data, blobKey);
+      try {
+        this.pFrame.addColumnSpec(column.id, column.spec);
+        this.pFrame.setColumnData(column.id, dataInfo);
+      } catch (err) {
+        throw new Error(
+          `Adding column ${column.id} to PFrame failed: ${err}; Spec: ${column.spec}, DataInfo: ${dataInfo}.`
+        );
+      }
+    }
+  }
+
+  private getOrCreateComputableForBlob(blobId: string) {
+    let computable = this.blobHandleComputables.get(blobId);
+    if (computable !== undefined) return computable;
+
+    const blobResource = this.blobIdToResource.get(blobId);
+    if (blobResource === undefined) throw new Error(`Blob with id ${blobId} not found.`);
+
+    // precalculation of value tree will trigger the download proecess right away
+    computable = this.blobDriver.getDownloadedBlob(blobResource).withPreCalculatedValueTree();
+
+    this.blobHandleComputables.set(blobId, computable);
+
+    return computable;
+  }
+
+  public readonly preloadBlob = async (blobIds: string[]): Promise<void> => {
+    const computables = blobIds.map((blobId) => this.getOrCreateComputableForBlob(blobId));
+    for (const computable of computables) await computable.awaitStableFullValue();
+  };
+
+  public readonly resolveBlob = async (blobId: string): Promise<string> => {
+    const computable = this.getOrCreateComputableForBlob(blobId);
+    return this.blobDriver.getLocalPath((await computable.awaitStableValue()).handle);
+  };
+
+  [Symbol.dispose](): void {
+    for (const computable of this.blobHandleComputables.values()) computable.resetState();
+    this.pFrame.dispose();
+  }
+}
+
+type FullPTableDef = {
+  pFrameHandle: PFrameHandle;
+  def: PTableDef<PObjectId>;
+};
+
+export class PFrameDriver implements SdkPFrameDriver {
+  private readonly pFrames: RefCountResourcePool<InternalPFrameData, PFrameHolder>;
+  private readonly pTables: RefCountResourcePool<FullPTableDef, Promise<PFrameInternal.PTable>>;
+
+  constructor(private readonly blobDriver: DownloadDriver) {
+    this.pFrames = new (class extends RefCountResourcePool<InternalPFrameData, PFrameHolder> {
+      constructor(private readonly blobDriver: DownloadDriver) {
+        super();
+      }
+      protected createNewResource(params: InternalPFrameData): PFrameHolder {
+        return new PFrameHolder(this.blobDriver, params);
+      }
+      protected calculateParamsKey(params: InternalPFrameData): string {
+        return stableKeyFromPFrameData(params);
+      }
+    })(this.blobDriver);
+
+    this.pTables = new (class extends RefCountResourcePool<
+      FullPTableDef,
+      Promise<PFrameInternal.PTable>
+    > {
+      constructor(
+        private readonly pFrames: RefCountResourcePool<InternalPFrameData, PFrameHolder>
+      ) {
+        super();
+      }
+      protected async createNewResource(params: FullPTableDef): Promise<PFrameInternal.PTable> {
+        const pFrame = this.pFrames.getByKey(params.pFrameHandle);
+        const rawPTable = await pFrame.pFrame.createTable({
+          src: joinEntryToInternal(params.def.src),
+          filters: params.def.filters
+        });
+        return params.def.sorting.length !== 0 ? rawPTable.sort(params.def.sorting) : rawPTable;
+      }
+      protected calculateParamsKey(params: FullPTableDef): string {
+        return stableKeyFromFullPTableDef(params);
+      }
+    })(this.pFrames);
+  }
+
+  //
+  // Internal / Config API Methods
+  //
+
+  public createPFrame(def: PFrameDef<PlTreeNodeAccessor>, ctx: ComputableCtx): PFrameHandle {
+    const internalData = def.map((c) => mapPObjectData(c, (d) => parseDataInfoResource(d)));
+    const res = this.pFrames.acquire(internalData);
+    ctx.addOnDestroy(res.unref);
+    return res.key as PFrameHandle;
+  }
+
+  public createPTable(
+    def: PTableDef<PColumn<PlTreeNodeAccessor>>,
+    ctx: ComputableCtx
+  ): PTableHandle {
+    const pFrameHandle = this.createPFrame(extractAllColumns(def.src), ctx);
+    const defIds = mapPTableDef(def, (c) => c.id);
+    const res = this.pTables.acquire({ def: defIds, pFrameHandle });
+    ctx.addOnDestroy(res.unref); // in addition to pframe unref added in createPFrame above
+    return res.key as PTableHandle;
+  }
+
+  //
+  // PFrame istance methods
+  //
+
+  public async findColumns(
+    handle: PFrameHandle,
+    request: FindColumnsRequest
+  ): Promise<FindColumnsResponse> {
+    const iRequest: PFrameInternal.FindColumnsRequest = {
+      ...request,
+      compatibleWith:
+        request.compatibleWith.length !== 0
+          ? [{ axesSpec: request.compatibleWith, qualifications: [] }]
+          : []
+    };
+    return {
+      hits: (await this.pFrames.getByKey(handle).pFrame.findColumns(iRequest)).hits.map(
+        (h) => h.hit
+      )
+    };
+  }
+
+  public async getColumnSpec(handle: PFrameHandle, columnId: PObjectId): Promise<PColumnSpec> {
+    return this.pFrames.getByKey(handle).pFrame.getColumnSpec(columnId);
+  }
+
+  public async listColumns(handle: PFrameHandle): Promise<PColumnIdAndSpec[]> {
+    return this.pFrames.getByKey(handle).pFrame.listColumns();
+  }
+
+  public async calculateTableData(
+    handle: PFrameHandle,
+    request: CalculateTableDataRequest<PObjectId>
+  ): Promise<CalculateTableDataResponse> {
+    let table = await this.pFrames.getByKey(handle).pFrame.createTable({
+      src: joinEntryToInternal(request.src),
+      filters: request.filters
+    });
+
+    if (request.sorting.length > 0) {
+      const sortedTable = await table.sort(request.sorting);
+      table.dispose();
+      table = sortedTable;
+    }
+
+    const spec = table.getSpec();
+    const data = await table.getData([...spec.keys()]);
+    table.dispose();
+
+    return spec.map((spec, i) => ({
+      spec: spec,
+      data: data[i]
+    }));
+  }
+
+  public async getUniqueValues(
+    handle: PFrameHandle,
+    request: UniqueValuesRequest
+  ): Promise<UniqueValuesResponse> {
+    return await this.pFrames.getByKey(handle).pFrame.getUniqueValues(request);
+  }
+
+  //
+  // PTable istance methods
+  //
+
+  public async getShape(handle: PTableHandle): Promise<PTableShape> {
+    const pTable = await this.pTables.getByKey(handle);
+    return pTable.getShape();
+  }
+
+  public async getSpec(handle: PTableHandle): Promise<PTableColumnSpec[]> {
+    const pTable = await this.pTables.getByKey(handle);
+    return pTable.getSpec();
+  }
+
+  public async getData(
+    handle: PTableHandle,
+    columnIndices: number[],
+    range?: TableRange
+  ): Promise<PTableVector[]> {
+    const pTable = await this.pTables.getByKey(handle);
+    return pTable.getData(columnIndices, range);
+  }
+}
+
+function joinEntryToInternal(entry: JoinEntry<PObjectId>): PFrameInternal.JoinEntry {
+  switch (entry.type) {
+    case 'column':
+      return {
+        type: 'column',
+        columnId: entry.column,
+        qualifications: []
+      };
+    case 'inner':
+    case 'full':
+      return {
+        type: entry.type,
+        entries: entry.entries.map((col) => joinEntryToInternal(col))
+      };
+    case 'outer':
+      return {
+        type: 'outer',
+        primary: joinEntryToInternal(entry.primary),
+        secondary: entry.secondary.map((col) => joinEntryToInternal(col))
+      };
+    default:
+      assertNever(entry);
+  }
+}
+
+function stableKeyFromFullPTableDef(data: FullPTableDef): string {
+  const hash = createHash('sha256');
+  hash.update(data.pFrameHandle);
+  hash.update(canonicalize(data.def)!);
+  return hash.digest().toString('hex');
+}
+
+function stableKeyFromPFrameData(data: PColumn<unknown>[]): string {
+  // PObject IDs derived from the PObjects canonical identity, so represents the content
+  const ids = data.map((d) => d.id).sort();
+  const hash = createHash('sha256');
+  let previous = '';
+  for (const id of ids) {
+    if (previous === id) continue; // only unique ids
+    hash.update(id);
+    previous = id;
+  }
+  return hash.digest().toString('hex');
+}
+
+export function extractAllColumns<D>(entry: JoinEntry<PColumn<D>>): PFrameDef<D> {
+  const columns = new Map<PObjectId, PColumn<D>>();
+  addAllColumns(entry, columns);
+  return [...columns.values()];
+}
+
+function addAllColumns<D>(entry: JoinEntry<PColumn<D>>, map: Map<PObjectId, PColumn<D>>): void {
+  switch (entry.type) {
+    case 'column':
+      map.set(entry.column.id, entry.column);
+      return;
+    case 'full':
+    case 'inner':
+      for (const e of entry.entries) addAllColumns(e, map);
+      return;
+    case 'outer':
+      addAllColumns(entry.primary, map);
+      for (const e of entry.secondary) addAllColumns(e, map);
+      return;
+    default:
+      assertNever(entry);
+  }
+}

package/src/pool/index.ts

@@ -0,0 +1,2 @@
+export * from './driver';
+export * from './result_pool';

package/src/pool/p_object_collection.ts

@@ -0,0 +1,122 @@
+import { PlError, PlTreeNodeAccessor } from '@milaboratories/pl-tree';
+import { PObject, PObjectSpec, ValueOrError } from '@platforma-sdk/model';
+import { notEmpty } from '@milaboratories/ts-helpers';
+import assert from 'assert';
+import { Writable } from 'utility-types';
+import { derivePObjectId } from './data';
+
+/** Represents specific staging or prod ctx data */
+export interface RawPObjectCollection {
+  /** true if no new results are expected */
+  readonly locked: boolean;
+  /** results by name */
+  readonly results: Map<string, RawPObjectEntry>;
+}
+
+/** Single result in a particular ctx */
+export interface RawPObjectEntry {
+  /**
+   * true - means this result has data field, however it may still be not ready
+   * false - means it can be derived that this result incarnation is spec-only
+   * undefined - means that it is not yet known
+   * */
+  readonly hasData?: boolean;
+
+  /** Result may be added even if there is no data associated with it */
+  readonly spec?: PObjectSpec;
+
+  /**
+   * Returns data accessor, or error, or undefined if data not yet available.
+   * If data fuinction itself is not defined, this means that corresponding context
+   * was not rendered.
+   * */
+  data?(): ValueOrError<PlTreeNodeAccessor, string> | undefined;
+}
+
+const BContextValuePrefix = 'values/';
+const BContextValueSpecSuffix = '.spec';
+const BContextValueDataSuffix = '.data';
+
+export function parseRawPObjectCollection(
+  node: PlTreeNodeAccessor,
+  errorOnUnknownField: boolean = true,
+  ignoreFieldErrors: boolean = false,
+  prefix: string = ''
+): RawPObjectCollection {
+  const entryPattern = /^(?<name>.*)\.(?<type>spec|data)$/;
+  const results = new Map<string, Writable<RawPObjectEntry>>();
+  for (const fieldName of node.listInputFields()) {
+    const match = fieldName.match(entryPattern);
+    if (!match) {
+      if (errorOnUnknownField) throw new Error(`unexpected field name ${fieldName}`);
+      else continue;
+    }
+
+    let name = notEmpty(match.groups?.name);
+    if (!name.startsWith(prefix)) {
+      if (errorOnUnknownField) throw new Error(`unexpected field name ${fieldName}`);
+      else continue;
+    }
+    name = name.slice(prefix.length);
+
+    const type = notEmpty(match.groups?.type) as 'spec' | 'data';
+    let result = results.get(name);
+    if (result === undefined) {
+      result = {};
+      results.set(name, result);
+    }
+
+    switch (type) {
+      case 'spec':
+        result.spec = node
+          .traverse({
+            field: fieldName,
+            ignoreError: ignoreFieldErrors,
+            pureFieldErrorToUndefined: ignoreFieldErrors
+          })
+          ?.getDataAsJson();
+        break;
+      case 'data':
+        result.hasData = true;
+        result.data = () =>
+          node.traverseOrError({
+            field: fieldName,
+            ignoreError: ignoreFieldErrors
+          });
+      default:
+        // other value types planned
+        continue;
+    }
+  }
+
+  const locked = node.getInputsLocked();
+  if (locked)
+    for (const [, result] of results) if (result.data === undefined) result.hasData = false;
+
+  return { locked, results };
+}
+
+export function parseFinalPObjectCollection(
+  node: PlTreeNodeAccessor,
+  errorOnUnknownField: boolean = true,
+  prefix: string = ''
+): Record<string, PObject<PlTreeNodeAccessor>> {
+  if (!node.getIsReadyOrError()) throw new Error('resource is not ready');
+  const rawCollection = parseRawPObjectCollection(node, errorOnUnknownField, false, prefix);
+  assert(rawCollection.locked);
+  const collection: Record<string, PObject<PlTreeNodeAccessor>> = {};
+  for (const [exportName, result] of rawCollection.results) {
+    if (result.spec === undefined) throw new Error(`no spec for key ${exportName}`);
+    if (result.hasData !== true || result.data === undefined)
+      throw new Error(`no data for key ${exportName}`);
+    const data = result.data();
+    if (data === undefined) throw new Error(`no data for key ${exportName}`);
+    if (!data.ok) throw new PlError(data.error);
+    collection[exportName] = {
+      id: derivePObjectId(result.spec, data.value),
+      spec: result.spec,
+      data: data.value
+    };
+  }
+  return collection;
+}