@milaboratories/pl-model-middle-layer 1.2.16

package/src/block_meta/content_conversion.ts

@@ -0,0 +1,13 @@
+import { ContentAbsoluteUrl, ContentAnyLocal, ContentRelative } from "./content_types";
+
+export function mapRemoteToAbsolute(
+  rootUrl: string
+): <T extends ContentAnyLocal>(
+  value: T
+) => Exclude<T, ContentRelative> | ContentAbsoluteUrl {
+  const rootWithSlash = rootUrl.endsWith('/') ? rootUrl : `${rootUrl}/`;
+  return <T extends ContentAnyLocal>(value: T) =>
+    value.type === 'relative'
+      ? { type: 'absolute-url', url: rootWithSlash + value.path }
+      : (value as Exclude<T, ContentRelative>);
+}

package/src/block_meta/content_types.ts

@@ -0,0 +1,233 @@
+import { z } from 'zod';
+
+//
+// Base content types
+//
+
+export const ContentExplicitString = z
+  .object({
+    type: z.literal('explicit-string'),
+    content: z.string().describe('Actual string value')
+  })
+  .strict();
+export type ContentExplicitString = z.infer<typeof ContentExplicitString>;
+
+export const ContentExplicitBase64 = z
+  .object({
+    type: z.literal('explicit-base64'),
+    mimeType: z
+      .string()
+      .regex(/\w+\/[-+.\w]+/)
+      .describe('MIME type to interpret content'),
+    content: z.string().base64().describe('Base64 encoded binary value')
+  })
+  .strict();
+export type ContentExplicitBase64 = z.infer<typeof ContentExplicitBase64>;
+
+export const ContentRelative = z
+  .object({
+    type: z.literal('relative'),
+    path: z
+      .string()
+      .describe(
+        'Address of the file, in most cases relative to the file which this structure is a part of'
+      )
+  })
+  .strict();
+export type ContentRelative = z.infer<typeof ContentRelative>;
+
+export const ContentAbsoluteFile = z
+  .object({
+    type: z.literal('absolute-file'),
+    file: z.string().startsWith('/').describe('Absolute address of the file in local file system')
+  })
+  .strict();
+export type ContentAbsoluteFile = z.infer<typeof ContentAbsoluteFile>;
+
+export const ContentAbsoluteUrl = z
+  .object({
+    type: z.literal('absolute-url'),
+    url: z.string().url().describe('Global URL to reach the requested file')
+  })
+  .strict();
+export type ContentAbsoluteUrl = z.infer<typeof ContentAbsoluteUrl>;
+
+//
+// Special content types
+//
+
+export const ContentExplicit = z
+  .object({
+    type: z.literal('explicit'),
+    content: z.instanceof(Uint8Array).describe('Raw content')
+  })
+  .strict();
+export type ContentExplicit = z.infer<typeof ContentExplicit>;
+
+export const ContentAbsoluteFolder = z
+  .object({
+    type: z.literal('absolute-folder'),
+    folder: z
+      .string()
+      .startsWith('/')
+      .describe('Absolute address of the folder in local file system')
+  })
+  .strict();
+export type ContentAbsoluteFolder = z.infer<typeof ContentAbsoluteFolder>;
+
+//
+// Unions
+//
+
+export const ContentAny = z.discriminatedUnion('type', [
+  ContentExplicitString,
+  ContentExplicitBase64,
+  ContentRelative,
+  ContentAbsoluteFile,
+  ContentAbsoluteUrl
+]);
+export type ContentAny = z.infer<typeof ContentAny>;
+
+export const ContentAnyLocal = z.discriminatedUnion('type', [
+  ContentExplicitString,
+  ContentExplicitBase64,
+  ContentRelative,
+  ContentAbsoluteFile
+]);
+export type ContentAnyLocal = z.infer<typeof ContentAnyLocal>;
+
+export const ContentAnyRemote = z.discriminatedUnion('type', [
+  ContentExplicitString,
+  ContentExplicitBase64,
+  ContentRelative,
+  ContentAbsoluteUrl
+]);
+export type ContentAnyRemote = z.infer<typeof ContentAnyRemote>;
+
+//
+// Narrow types with relative option
+//
+
+// export const ContentAnyBinaryRemote = z.discriminatedUnion('type', [
+//   ContentExplicitBase64,
+//   ContentRelative,
+//   ContentAbsoluteUrl
+// ]);
+// export type ContentAnyBinaryRemote = z.infer<typeof ContentAnyBinaryRemote>;
+
+export const ContentAnyBinaryLocal = z.discriminatedUnion('type', [
+  ContentExplicitBase64,
+  ContentRelative,
+  ContentAbsoluteFile
+]);
+export type ContentAnyBinaryLocal = z.infer<typeof ContentAnyBinaryLocal>;
+
+// export const ContentAnyTextRemote = z.discriminatedUnion('type', [
+//   ContentExplicitString,
+//   ContentRelative,
+//   ContentAbsoluteUrl
+// ]);
+// export type ContentAnyTextRemote = z.infer<typeof ContentAnyTextRemote>;
+
+export const ContentAnyTextLocal = z.discriminatedUnion('type', [
+  ContentExplicitString,
+  ContentRelative,
+  ContentAbsoluteFile
+]);
+export type ContentAnyTextLocal = z.infer<typeof ContentAnyTextLocal>;
+
+//
+// Narrow absolute types
+//
+
+export const ContentAbsoluteBinaryRemote = z.discriminatedUnion('type', [
+  ContentExplicitBase64,
+  ContentAbsoluteUrl
+]);
+export type ContentAbsoluteBinaryRemote = z.infer<typeof ContentAbsoluteBinaryRemote>;
+
+export const ContentAbsoluteBinaryLocal = z.discriminatedUnion('type', [
+  ContentExplicitBase64,
+  ContentAbsoluteFile
+]);
+export type ContentAbsoluteBinaryLocal = z.infer<typeof ContentAbsoluteBinaryLocal>;
+
+export const ContentAbsoluteTextRemote = z.discriminatedUnion('type', [
+  ContentExplicitString,
+  ContentAbsoluteUrl
+]);
+export type ContentAbsoluteTextRemote = z.infer<typeof ContentAbsoluteTextRemote>;
+
+export const ContentAbsoluteTextLocal = z.discriminatedUnion('type', [
+  ContentExplicitString,
+  ContentAbsoluteFile
+]);
+export type ContentAbsoluteTextLocal = z.infer<typeof ContentAbsoluteTextLocal>;
+
+//
+// Narrow relative types
+//
+
+export const ContentRelativeBinary = z.discriminatedUnion('type', [
+  ContentExplicitBase64,
+  ContentRelative
+]);
+export type ContentRelativeBinary = z.infer<typeof ContentRelativeBinary>;
+
+export const ContentRelativeText = z.discriminatedUnion('type', [
+  ContentExplicitString,
+  ContentRelative
+]);
+export type ContentRelativeText = z.infer<typeof ContentRelativeText>;
+
+// export function ConstructContent(
+//   contentType: 'text',
+//   contextType: 'local'
+// ): typeof ContentAnyTextLocal;
+// export function ConstructContent(
+//   contentType: 'text',
+//   contextType: 'remote'
+// ): typeof ContentAnyTextRemote;
+// export function ConstructContent(
+//   contentType: 'binary',
+//   contextType: 'local'
+// ): typeof ContentAnyBinaryLocal;
+// export function ConstructContent(
+//   contentType: 'binary',
+//   contextType: 'remote'
+// ): typeof ContentAnyBinaryRemote;
+// export function ConstructContent(
+//   contentType: ContentType,
+//   contextType: ContextType
+// ):
+//   | typeof ContentAnyTextLocal
+//   | typeof ContentAnyTextRemote
+//   | typeof ContentAnyBinaryLocal
+//   | typeof ContentAnyBinaryRemote;
+// export function ConstructContent(contentType: ContentType, contextType: ContextType) {
+//   return contentType === 'text'
+//     ? contextType === 'local'
+//       ? ContentAnyTextLocal
+//       : ContentAnyTextRemote
+//     : contextType === 'local'
+//       ? ContentAnyBinaryLocal
+//       : ContentAnyBinaryRemote;
+// }
+
+export const DescriptionContentBinary = z.union([
+  z
+    .string()
+    .startsWith('file:')
+    .transform<ContentRelativeBinary>((value, ctx) => ({ type: 'relative', path: value.slice(5) })),
+  ContentAnyBinaryLocal
+]);
+export type DescriptionContentBinary = z.infer<typeof DescriptionContentBinary>;
+
+export const DescriptionContentText = z.union([
+  z.string().transform<ContentRelativeText>((value, ctx) => {
+    if (value.startsWith('file:')) return { type: 'relative', path: value.slice(5) };
+    else return { type: 'explicit-string', content: value };
+  }),
+  ContentAnyTextLocal
+]);
+export type DescriptionContentText = z.infer<typeof DescriptionContentText>;

package/src/block_meta/index.ts

@@ -0,0 +1,51 @@
+import { ZodTypeAny, z } from 'zod';
+import {
+  BlockComponentsDescriptionRaw,
+  BlockComponentsManifest
+} from './block_components';
+import { BlockPackId } from './block_pack_id';
+import {
+  BlockPackMetaDescriptionRaw,
+  BlockPackMetaManifest
+} from './meta';
+
+export * from './block_components'
+export * from './block_pack_id'
+export * from './common'
+export * from './content_types'
+export * from './meta'
+export * from './semver'
+
+export const BlockPackDescriptionFromPackageJsonRaw = z.object({
+  components: BlockComponentsDescriptionRaw,
+  meta: BlockPackMetaDescriptionRaw
+});
+
+export function CreateBlockPackDescriptionSchema<Components extends ZodTypeAny, Meta extends ZodTypeAny>(
+  components: Components,
+  meta: Meta
+) {
+  return z.object({
+    id: BlockPackId,
+    components,
+    meta
+  });
+}
+
+export const BlockPackDescriptionManifest = CreateBlockPackDescriptionSchema(
+  BlockComponentsManifest,
+  BlockPackMetaManifest
+);
+export type BlockPackDescriptionManifest = z.infer<typeof BlockPackDescriptionManifest>;
+
+export const BlockPackDescriptionRaw = CreateBlockPackDescriptionSchema(
+  BlockComponentsDescriptionRaw,
+  BlockPackMetaDescriptionRaw
+);
+export type BlockPackDescriptionRaw = z.infer<typeof BlockPackDescriptionRaw>;
+
+export const BlockPackManifest = BlockPackDescriptionManifest.extend({
+  schema: z.literal('v1'),
+  files: z.array(z.string())
+});
+export type BlockPackManifest = z.infer<typeof BlockPackManifest>;

package/src/block_meta/meta.ts

@@ -0,0 +1,54 @@
+import { z } from 'zod';
+import {
+  ContentAbsoluteBinaryLocal,
+  ContentAbsoluteTextLocal,
+  ContentExplicitBase64,
+  ContentExplicitString,
+  ContentRelative,
+  ContentRelativeBinary,
+  ContentRelativeText,
+  DescriptionContentBinary,
+  DescriptionContentText
+} from './content_types';
+
+export function BlockPackMeta<
+  const LongStringType extends z.ZodTypeAny,
+  const BinaryType extends z.ZodTypeAny
+>(longString: LongStringType, binary: BinaryType) {
+  return z.object({
+    title: z.string(),
+    description: z.string(),
+    longDescription: longString.optional(),
+    logo: binary.optional(),
+    url: z.string().url().optional(),
+    docs: z.string().url().optional(),
+    support: z.union([z.string().url(), z.string().email()]).optional(),
+    tags: z.array(z.string()).optional(),
+    organization: z.object({
+      name: z.string(),
+      url: z.string().url(),
+      logo: binary.optional()
+    })
+  });
+}
+
+// prettier-ignore
+export const BlockPackMetaDescriptionRaw = BlockPackMeta(
+  DescriptionContentText,
+  DescriptionContentBinary
+);
+export type BlockPackMetaDescriptionRaw = z.infer<typeof BlockPackMetaDescriptionRaw>;
+
+// prettier-ignore
+export const BlockPackMetaManifest = BlockPackMeta(
+  ContentRelativeText,
+  ContentRelativeBinary
+);
+export type BlockPackMetaManifest = z.infer<typeof BlockPackMetaManifest>;
+
+// prettier-ignore
+export const BlockPackMetaEmbeddedContent = BlockPackMeta(
+  z.string(),
+  ContentExplicitBase64
+);
+export type BlockPackMetaEmbeddedContent = z.infer<typeof BlockPackMetaEmbeddedContent>;

package/src/block_meta/semver.ts

@@ -0,0 +1,10 @@
+import { z } from 'zod';
+
+// Regex taken from here:
+//   https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
+export const SemVer = z
+  .string()
+  .regex(
+    /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/,
+    'Wrong version format, please use valid semver'
+  );

package/src/block_pack.ts

@@ -0,0 +1,29 @@
+/** Block pack from local folder, to be used during block development. Legacy layout. */
+export interface BlockPackDevV1 {
+  type: 'dev' | 'dev-v1';
+  folder: string;
+  mtime?: string;
+}
+
+/** Block pack from local folder, to be used during block development. New layout. */
+export interface BlockPackDevV2 {
+  type: 'dev-v2';
+  folder: string;
+  mtime?: string;
+}
+
+/** Block pack from registry with version 1 layout, to be loaded directly
+ * from the client. */
+export interface BlockPackFromRegistryV1 {
+  type: 'from-registry-v1';
+  registryUrl: string;
+  organization: string;
+  package: string;
+  version: string;
+}
+
+/** Information about block origin, can be used to instantiate new blocks */
+export type BlockPackSpec =
+  | BlockPackDevV1
+  | BlockPackDevV2
+  | BlockPackFromRegistryV1;

package/src/block_state.ts

@@ -0,0 +1,14 @@
+import { BlockOutputsBase, BlockState } from '@milaboratories/pl-model-common';
+import { AuthorMarker } from './author_marker';
+import { Optional } from 'utility-types';
+
+export type BlockStateInternal<
+  Args = unknown,
+  Outputs extends BlockOutputsBase = BlockOutputsBase,
+  UiState = unknown,
+  Href extends `/${string}` = `/${string}`
+> = Optional<Readonly<BlockState<Args, Outputs, UiState, Href>>, 'outputs'> & {
+  /** Author marker for Args and UI State. Absence of author marker, means that
+   * last modifier of the state provided no author marker. */
+  readonly author?: AuthorMarker;
+};

package/src/index.ts

@@ -0,0 +1,8 @@
+export * from './author_marker';
+export * from './block_pack';
+export * from './block_state';
+export * from './block_meta';
+export * from './project';
+export * from './project_list';
+export * from './project_overview';
+export * from './pframe';

package/src/pframe/index.ts

@@ -0,0 +1 @@
+export * as PFrameInternal from './internal_api';

package/src/pframe/internal_api/api_factory.ts

@@ -0,0 +1,68 @@
+import { PColumnSpec, PObjectId } from '@milaboratories/pl-model-common';
+
+/** Abstract identifier of a data blob that can be requested from the storage backend */
+export type PFrameBlobId = string;
+
+export type JsonDataValue = string | number | null;
+
+export type JsonDataInfo = {
+  type: 'Json';
+  keyLength: number;
+  data: Record<string, JsonDataValue>;
+};
+
+export type JsonPartitionedDataInfo<Blob = PFrameBlobId> = {
+  type: 'JsonPartitioned';
+  partitionKeyLength: number;
+  parts: Record<string, Blob>;
+};
+
+export type BinaryChunkInfo<Blob = PFrameBlobId> = {
+  index: Blob;
+  values: Blob;
+};
+
+export type BinaryPartitionedDataInfo<Blob = PFrameBlobId> = {
+  type: 'BinaryPartitioned';
+  partitionKeyLength: number;
+  parts: Record<string, BinaryChunkInfo<Blob>>;
+};
+
+export type DataInfo<Blob = PFrameBlobId> =
+  | JsonDataInfo
+  | JsonPartitionedDataInfo<Blob>
+  | BinaryPartitionedDataInfo<Blob>;
+
+/** Path of the file containing requested data (blob). This path is returned by
+ * {@link BlobPathResolver} as soon as blob materialized in the file system. */
+export type FilePath = string;
+
+/** Data source allows PFrame to retrieve the data blobs for columns with assigned data info. */
+export type PFrameDataSource = {
+  /**
+   * PFrame may notify storage backend about its plans to use particular blobs in the future.
+   * Storage backend will do its best to preload specified blob so the subsequent
+   * {@link resolveBlob} will quickly return preloaded file path.
+   */
+  preloadBlob(blobIds: PFrameBlobId[]): Promise<void>;
+
+  /** Allows to read actual data given the blob id from {@link DataInfo}. */
+  resolveBlob(blobId: PFrameBlobId): Promise<FilePath>;
+};
+
+/** API exposed by PFrames library allowing to create and provide data for
+ * PFrame objects */
+export interface PFrameFactoryAPI {
+  /** Associates data source with this PFrame */
+  setDataSource(dataSource: PFrameDataSource): void;
+
+  /** Adds PColumn without spec */
+  addColumnSpec(columnId: PObjectId, columnSpec: PColumnSpec): void;
+
+  /** Associates data info with cpecific column */
+  setColumnData(columnId: PObjectId, dataInfo: DataInfo): void;
+
+  /** Releases all the data previously added to PFrame using methods above,
+   * any interactions with disposed PFrame will result in exception */
+  dispose(): void;
+}

package/src/pframe/internal_api/api_read.ts

@@ -0,0 +1,39 @@
+import { FindColumnsRequest, FindColumnsResponse } from './find_columns';
+import { DeleteColumnFromColumnsRequest, DeleteColumnFromColumnsResponse } from './delete_column';
+import {
+  PColumnInfo,
+  PColumnSpec,
+  PObjectId,
+  UniqueValuesRequest,
+  UniqueValuesResponse
+} from '@milaboratories/pl-model-common';
+import { CreateTableRequest } from './create_table';
+import { PTable } from './table';
+
+/** Read interface exposed by PFrames library */
+export interface PFrameReadAPI {
+  /**
+   * Finds columns given filtering criteria on column name, annotations etc.
+   * and a set of qualified axes specs to find only columns with compatible
+   * axes spec.
+   *
+   * Only column specs are used, this method will work even for columns
+   * with no assigned data.
+   * */
+  findColumns(request: FindColumnsRequest): Promise<FindColumnsResponse>;
+
+  /** To be reviewed in the future */
+  deleteColumn(request: DeleteColumnFromColumnsRequest): Promise<DeleteColumnFromColumnsResponse>;
+
+  /** Retrieve single column spec */
+  getColumnSpec(columnId: PObjectId): Promise<PColumnSpec>;
+
+  /** Retrieve information about all columns currently added to the PFrame */
+  listColumns(): Promise<PColumnInfo[]>;
+
+  /** Calculates data for the table and returns an object to access it */
+  createTable(request: CreateTableRequest): Promise<PTable>;
+
+  /** Calculate set of unique values for a specific axis for the filtered set of records */
+  getUniqueValues(request: UniqueValuesRequest): Promise<UniqueValuesResponse>;
+}

package/src/pframe/internal_api/common.ts

@@ -0,0 +1,22 @@
+import { AxisId, AxisSpec, ValueType } from '@milaboratories/pl-model-common';
+
+export interface SingleAxisSelector {
+  name: string;
+  type?: ValueType;
+  domain?: Record<string, string>;
+}
+
+export interface AxisQualification {
+  axis: SingleAxisSelector;
+  additionalDomains: Record<string, string>;
+}
+
+export interface AxisQualificationWithAxisId {
+  axis: AxisId;
+  additionalDomains: Record<string, string>;
+}
+
+export interface ColumnAxesWithQualifications {
+  axesSpec: AxisSpec[];
+  qualifications: AxisQualification[];
+}

package/src/pframe/internal_api/create_table.ts

@@ -0,0 +1,31 @@
+import { AxisQualificationWithAxisId } from './common';
+import { PObjectId, PTableRecordFilter } from '@milaboratories/pl-model-common';
+
+export interface ColumnJoinEntry {
+  type: 'column';
+  columnId: PObjectId;
+  qualifications: AxisQualificationWithAxisId[];
+}
+
+export interface InnerJoin {
+  type: 'inner';
+  entries: JoinEntry[];
+}
+
+export interface FullJoin {
+  type: 'full';
+  entries: JoinEntry[];
+}
+
+export interface OuterJoin {
+  type: 'outer';
+  primary: JoinEntry;
+  secondary: JoinEntry[];
+}
+
+export type JoinEntry = ColumnJoinEntry | InnerJoin | FullJoin | OuterJoin;
+
+export interface CreateTableRequest {
+  src: JoinEntry;
+  filters: PTableRecordFilter[];
+}

package/src/pframe/internal_api/delete_column.ts

@@ -0,0 +1,12 @@
+import { ColumnAxesWithQualifications } from './common';
+
+export interface DeleteColumnFromColumnsRequest {
+  columns: ColumnAxesWithQualifications[];
+
+  /** Zero based index of the column to delete */
+  delete: number;
+}
+
+export interface DeleteColumnFromColumnsResponse {
+  columns: ColumnAxesWithQualifications[];
+}

package/src/pframe/internal_api/find_columns.ts

@@ -0,0 +1,27 @@
+import { ColumnFilter, PColumnIdAndSpec } from '@milaboratories/pl-model-common';
+import { AxisQualification, ColumnAxesWithQualifications } from './common';
+
+export interface FindColumnsRequest {
+  columnFilter: ColumnFilter;
+  compatibleWith: ColumnAxesWithQualifications[];
+  strictlyCompatible: boolean;
+}
+
+export interface FindColumnResponseQualifications {
+  forQueries: AxisQualification[][];
+  forHit: AxisQualification[];
+}
+
+export interface FindColumnsMappingVariant {
+  qualifications: FindColumnResponseQualifications;
+  distinctiveQualifications: FindColumnResponseQualifications;
+}
+
+export interface FindColumnsResponseHit {
+  hit: PColumnIdAndSpec;
+  mappingVariants: FindColumnsMappingVariant[];
+}
+
+export interface FindColumnsResponse {
+  hits: FindColumnsResponseHit[];
+}

package/src/pframe/internal_api/index.ts

@@ -0,0 +1,9 @@
+export * from './api_read';
+export * from './api_factory';
+export * from './common';
+export * from './create_table';
+export * from './delete_column';
+export * from './find_columns';
+export * from './table';
+
+export * from './pframe';

package/src/pframe/internal_api/pframe.ts

@@ -0,0 +1,4 @@
+import { PFrameReadAPI, PFrameFactoryAPI } from './index';
+
+export interface PFrame extends PFrameFactoryAPI, PFrameReadAPI {
+}