dataset-types 3.4.2 → 3.5.0

package/src/types/index.ts

@@ -0,0 +1,738 @@
+import {
+  //Vector/Matrix
+
+  IVector,
+  IMatrix,
+
+  // Value Data
+  IExpression,
+  IValueMetadata,
+  IValueData,
+  IValueMatrix,
+
+  //Neuron
+  INeuron,
+
+  //Network
+  INetwork,
+  INNValue,
+  INNLayer,
+  IValueNetworkDisplayOption,
+  INNMatrix,
+
+  //Architecture Specification
+  INNSpecification,
+  IMatrixSpecification,
+
+  //Learning
+  INNTrainable,
+  ITrainingExample,
+  IOptimizer,
+  ILearningConfig,
+  ILearnable,
+  IEventCollector,
+  ITrainingDataSet,
+
+  // Transformer
+  ITransformerSpecification,
+  IValueTransformer,
+  IValueTransformerInstance,
+
+
+} from './valuedata'
+
+import {
+  // Attention
+  IAttention2Model,  
+  IAttention2ModelFactory,
+  IForwardResult,  
+  IModelSpecification,
+  IEmbedding,
+  IEmbeddingRecord,
+  INamedEmbedding,
+  ISequenceEmbeddings,
+  ISequenceMatrix,  
+  ITransformerExample,
+  ITransformerModel,
+  IDenseLayer, IMLP, ISGD, 
+  ITransformerTraining,
+  ITrainData, ITrainer,
+  IAttentionModelInputs,
+  IAttentionModelContextInputs
+
+} from './attention2'
+
+import {  
+  INode,  
+  ILink,
+  ITopology,  
+  INodeLink,
+  NODETYPE_TYPE
+  
+} from './topology'
+
+export {
+  IVector,
+  IMatrix,
+
+  // Value Data
+  IExpression,
+  IValueMetadata,
+  IValueData,
+  IValueMatrix,
+
+  //Neuron
+  INeuron,
+
+  //Network
+  INetwork,
+  INNValue,
+  INNLayer,
+  IValueNetworkDisplayOption,
+  INNMatrix,
+
+  //Architecture Specification
+  INNSpecification,
+  IMatrixSpecification,
+
+  //Learning
+  INNTrainable,
+  ITrainingExample,
+  IOptimizer,
+  ILearningConfig,
+  ILearnable,
+  IEventCollector,
+  ITrainingDataSet,
+
+  // Transformer
+  ITransformerSpecification,
+  IValueTransformer,
+  IValueTransformerInstance,
+
+  // Attention
+  IAttention2Model,  
+  IAttention2ModelFactory,
+  IForwardResult,  
+  IModelSpecification,
+  IEmbedding,
+  IEmbeddingRecord,
+  INamedEmbedding,
+  ISequenceEmbeddings,
+  ISequenceMatrix,  
+  ITransformerExample,
+  ITransformerModel,
+  ITransformerTraining,
+  IDenseLayer, IMLP, ISGD, ITrainData, ITrainer,
+  IAttentionModelInputs,
+  IAttentionModelContextInputs,
+
+  //Node Type
+  INode,  
+  ILink,
+  ITopology,  
+  INodeLink,
+  NODETYPE_TYPE
+}
+
+type BufferEncoding =
+  | "ascii"
+  | "utf8"
+  | "utf-8"
+  | "utf16le"
+  | "ucs2"
+  | "ucs-2"
+  | "base64"
+  | "latin1"
+  | "binary"
+  | "hex";
+
+export interface ICommandResult {
+  status: string;
+  msg: string;
+  entity?: any;
+  log?: any[];
+}
+
+export interface IResults<Type> {
+  isLoading: boolean;
+  rows: Type[];
+  pageInfo?: any;
+}
+
+export interface IContext {
+  context: string;
+  projectcontext: string;
+}
+export interface IName {
+  name: string;
+}
+
+export interface IDataRow {
+  id: string;
+}
+
+export interface INamedDataRow extends IDataRow, IName { }
+
+export interface IEntity extends IDataRow {
+  type: string;
+  delete?: string;
+}
+
+export interface INamedEntity extends IEntity, IName, IContext { }
+
+export interface ITimed {
+  createdBy: string;
+  createdAt: Date | string;
+  effective?: string;
+}
+
+export interface IDocumentReference extends IDataRow {
+  filetype: string;
+  documenttype: string;
+}
+
+export interface IDocument extends INamedEntity, ITimed {
+  entity: string;
+  filetype: string;
+  documenttype: string;
+  title?: string;
+  authoringcompany?: string;
+}
+
+export interface IDocumentEntityLink {
+  entity: string;
+  documentId: string;
+  documentName?: string;
+}
+
+export interface IConfigurationItem extends INamedEntity, ITimed {
+  itemtype: string;
+  active: boolean;
+  level: string;
+  monitored: string;
+  baseline: string;
+  dimension: string;
+  datatypes: string;
+  group?: string; //deprecated as this doesnt work with Duck DB
+  grouping: string;
+}
+
+export type IPermission = String;
+
+export interface IRolePermission {
+  permission: string;
+  role: string;
+}
+
+export interface IDataService { }
+
+export interface IModel {
+  close: any;
+}
+
+export interface IModelService {
+  model: IModel;
+  seed: any;
+  upsert: (type: string, entity: any[]) => Promise<IPersistResult>;
+  upsertBatch: (type: string, entity: any[]) => Promise<IPersistResult>;
+}
+
+export interface IColumn {
+  name: string;
+  displayName?: string;
+  tooltip?: string;
+  type: string;
+  label: string;
+  options: string[];
+  valueRange?: string;
+  default?: string;
+  derivation?: string;
+  required?: boolean;
+  distinct?: boolean;
+  width?: string;
+  truncate?: boolean;
+  defaultValue?: string | number;
+  groupName?: string;
+}
+
+export interface IColumnGroup {
+  name: string;
+  columns: string[];
+}
+
+export interface ITable {
+  file: string;
+  name: string;
+  dataname?: string;
+  options: string[];
+  rows?: string[];
+  columns: IColumn[];
+  columnGroups?: IColumnGroup[];
+  helpFileName?: string;
+}
+
+export interface IFileResult {
+  content: string;
+  contenttype: string;
+  filename: string;
+  metadata: any;
+}
+
+export interface IFileService {
+  download2: (
+    tenant: string,
+    projectcontext: string,
+    log: boolean,
+    txmode: string, //production/development/test
+    filetype: string,
+    level: string,
+    reference: string,
+    encoding: BufferEncoding, //base64,utf-8
+  ) => Promise<IFileResult>;
+  downloadToFile: (
+    localfilename: string,
+    tenantid: string,
+    projectcontext: string,
+    log: boolean,
+    filetype: string,
+    level: string,
+    reference: string,
+    txmode: string, //production/development/test
+  ) => Promise<IFileResult>;
+  sendBlock: (
+    tenantid: string,
+    projectcontext: string,
+    log: boolean, //NOLOG or LOG file.  This results in a different container
+    txmode: string, //production/development/test
+    filetype: string, //CSV, PDF,...
+    level: string, //Whether the files are stored at the LEVEL_TENANT or LEVEL_PROJECTCONTEXT
+    content: string, //This content to save
+    key: string, //blob id or filename
+    sliceNumber: number,
+    totalSlices: number,
+    meta: any, //meta data
+    tags: any, //tags
+  ) => void;
+  copy: (
+    sourceContainer: string,
+    sourceFile: string,
+    targetContainer: string,
+    targetFile: string,
+  ) => Promise<void>;
+  list: (
+    tenantid: string,
+    projectcontext: string,
+    log: boolean,
+    page: number,
+    pageSize: number,
+    term: string,
+    mode: string,
+    recordtype: string,
+    level: string,
+    enabled: boolean,
+    order: string,
+  ) => Promise<IDatasetFiles>;
+  remove: (
+    tenantid: string,
+    projectcontext: string,
+    level: string,
+    mode: string,
+    name: string,
+  ) => Promise<boolean>;
+  setMeta: (
+    tenantid: string,
+    projectcontext: string,
+    level: string,
+    mode: string,
+    name: string,
+    status: string,
+  ) => Promise<boolean>;
+  getContainerName: (
+    tenantid: string,
+    projectcontext: string,
+    level: string,
+    log: boolean,
+    txMode: string,
+  ) => string;
+}
+
+export interface ILogService {
+  archive: (
+    context: string,
+    projectcontext: string,
+    level: string,
+    submissionid: string,
+    username: string,
+    log: ILog[],
+    processingtype?: string,
+  ) => Promise<string>;
+  list: () => Promise<string[]>;
+}
+
+export interface ISecurityMonitorService {
+  hasPermission: (
+    permission: string,
+    roles: string[],
+    context: string,
+  ) => Promise<boolean>;
+}
+
+export type IChannel = "Email" | "SMS" | "WHATSAPP"; //How to refer to domain constants
+
+export interface IProcessingConfig {
+  remove?: boolean; //controls if the file is to be processed as a delete
+  record?: boolean; //Controls if files are uploaded to storage account. Used when replaying files. Used during unit tests to skip upload.
+  modes?: IChannel[];
+  notificationenabled?: boolean;
+}
+
+export interface IPersistResult {
+  data: any[] | any;
+  error?: string;
+}
+
+export interface ILogStatus {
+  archiveid: string;
+  error?: string;
+}
+
+export interface ILog {
+  processingType: string;
+  processingConfig?: IProcessingConfig;
+  //An ILog represents the information collected while handling a file
+  blobid?: string; //id in storage account
+  documentId?: string; //when log is submitted from browser it can have a document id (TODO make this just id)
+  name?: string; //Original name of the document when originally loaded
+  localfilename?: string; //filename on node process
+  user: string; //upn of user
+  meta: {
+    documenttype: string; //Record or Data Type
+    context: string; //Tenant Id
+    projectcontext: string; //Project Id
+    entity?: string; //Entity Id which contains this document
+    recordstatus?: string;
+    level?: string;
+    user?: string;
+    createdat?: string;
+    effective?: string;
+  };
+
+  document?: IDocument; //The document details
+  processing?: any[];
+  rawdata?: any[];
+  transformeddata?: any[];
+  content_mappings?: any[]; //Mappings from csv reference to blobid
+  enriched_contentmappings?: any[]; //DocumentDataLinks reified
+  enricheddata?: any[]; //Enriched Data extracted from the document
+  persisted_data_result?: any[];
+  classifieddata?: any[];
+  filetype?: string; //csv,pdf or other
+  filter?: any; //Used to associate filtering metadata
+  error?: string;
+  archiveid?: string;
+}
+
+export interface IUser {
+  id: string;
+  name: string;
+  email: string;
+  mobile: string;
+  roles: string;
+  context: string;
+}
+
+export type IDatasetFile = {
+  context: string;
+  id: string;
+  filename: string;
+  lastModified: string;
+  filetype: string;
+  recordtype: string;
+  entity: string;
+};
+
+export type IDatasetFiles = {
+  count: number;
+  files: IDatasetFile[];
+};
+
+export type IDayRange = number[];
+
+export interface IDataLockable extends IEntity {
+  locked: boolean;
+  lockedBy?: string;
+}
+export interface IDataArchivable extends IEntity {
+  archived: boolean;
+  archivedBy?: string;
+}
+
+export type IProject = IDataLockable &
+  IDataArchivable &
+  IName & {
+    description: string;
+    updatedBy: string;
+    updatedAt: Date;
+    createdBy: string;
+    createdAt: Date;
+    Categories?: ICategory[];
+  };
+export interface ICategory extends INamedEntity {
+  colour: number;
+  // Projects?: IProject[];
+}
+
+export interface IAction {
+  name: (item: any) => string; // eslint-disable-line
+  icon: (item: any) => string; // eslint-disable-line
+  action: (item: any) => void; // eslint-disable-line
+}
+
+export interface IActionContext {
+  callback: () => void;
+  permissions: IPermission[];
+  logidsSet: (logs: ILogStatus[]) => void;
+}
+
+export type IGridAction<Type> = (
+  d: Type,
+  ac: IActionContext | null,
+) => Promise<boolean>;
+export type IGridActionDisabled<Type> = (
+  d: Type,
+  ac: IActionContext | null,
+) => boolean;
+
+export interface IDataGridAction<Type> {
+  label: string;
+  disabled?: IGridActionDisabled<Type>;
+  onClick?: IGridAction<Type>;
+}
+
+//----Dataset Manage Data Set----
+
+export type API_POST_FILE_SLICE = (
+  filetype: string,
+  content: string | ArrayBuffer,
+  name: string,
+  documenttype: string,
+  level: string,
+  sliceNumber: number,
+  totalSlices: number,
+  meta: any,
+) => Promise<ICommandResult>;
+
+export interface IManageDataSetConfiguration {
+  context: string;
+  projectcontext: string;
+  name: string;
+  datatype: string;
+  datatypeSet: (type: string) => void;
+  datatype_meta?: IDataTypeMeta[];
+
+  columns: { [datatype: string]: any[] };
+
+  required_permission: string;
+  permissions: IPermission[];
+
+  actions?: IDataGridAction<IDocument>[];
+  dataactions?: { [datatype: string]: IDataGridAction<any>[] };
+
+  cleantransform?: ICleanTransformCallback;
+
+  filter?: {
+    name: string;
+    filter: IFilterCallback;
+  };
+
+  idtransform: (d: any) => any;
+  getIDsforData?: IGetIdsForDataCallback;
+  getDataByID?: IGetDataByIdsCallback;
+  combineExistingAndUpdate?: ICombineExistingAndUpdateCallback;
+  filechooser?: IFileChoiceCallback;
+
+  createDocument?: (d: IDocument, stuff: any) => Promise<ICommandResult>;
+  queryValidationData?: (
+    datatype: string,
+    entities: string[],
+  ) => Promise<ICommandResult>;
+  queryRecords?: (
+    documenttype: string,
+    datatype: string,
+    encoding: string,
+  ) => Promise<ICommandResult>;
+
+  postFileSlice?: API_POST_FILE_SLICE;
+  postReplay: { [datatype: string]: API_POST_REPLAY };
+
+  useConfigurationItems?: (
+    context: string,
+    itemtype: string,
+    hash: number,
+    page: number,
+    pageSize: number,
+  ) => IResults<IConfigurationItem>;
+  useDocuments?: (
+    context: string,
+    projectcontext: string,
+    associated: string[],
+    entity_type: string,
+    entities: string[],
+    entity: string,
+    filetype: string,
+    search: string,
+    hash: number,
+    page: number,
+    pageSize: number,
+    timeperiod: IDayRange | null,
+  ) => IResults<IDocument>;
+  // useValidation: (doctype: string) => IResults<any>
+  useData: { [datatype: string]: HOOK_USE_DATA };
+  validators: any;
+  groupColumns?: (dt: IDatasetTable, b: boolean) => ITable | undefined;
+  datatypeLevels?: {
+    //Levels (Tenant/Project) for data types such that files are saved to the right containers
+    [datatype: string]: string;
+  };
+}
+
+export type API_POST_SAMPLE = (source: string) => Promise<ICommandResult>;
+export type API_POST_REPLAY = (
+  source: string,
+  logs: ILog[],
+) => Promise<ICommandResult>;
+export type HOOK_USE_DATA = (
+  context: string,
+  projectcontext: string,
+  datatype: string | undefined,
+  documenttype: string | undefined,
+  entity: string,
+  entities: string[],
+  timeperiod: IDayRange | null,
+  hash: number,
+  page: number,
+  pageSize: number,
+  transform: ((d: any) => any) | null,
+  allentities: boolean,
+  where: any,
+) => IResults<any>;
+
+export interface IDataTypeMeta {
+  property: string;
+  description: string;
+  label: string;
+  tooltip: string;
+  value: (ci: IConfigurationItem) => string;
+}
+
+// ----Dataset Replay---
+export type HOOK_USE_FILES = (
+  context: string,
+  projectcontext: string,
+  term: string,
+  recordtype: string,
+  onlyenabled: boolean,
+  level: string,
+  hash: number,
+  page: number,
+  pageSize: number,
+) => IResults<any>;
+
+export type POST_SUBMIT_SAMPLE = () => any;
+
+export interface IManageDatasetReplayConfiguration {
+  context: string;
+  projectcontext: string;
+  name: string;
+  columns: { [datatype: string]: any[] };
+
+  required_permission: string;
+  permissions: IPermission[];
+
+  role: string;
+
+  actions?: IDataGridAction<IDocument>[];
+  dataactions?: { [datatype: string]: IDataGridAction<any>[] };
+  postSample: API_POST_SAMPLE;
+  postReplay: { [datatype: string]: API_POST_REPLAY };
+  useFiles?: { [datatype: string]: HOOK_USE_FILES };
+
+  customReplay?: {
+    name: string;
+    process: (log: ILog, base64content: string) => Promise<ILog[]>;
+  };
+}
+
+export interface IMenuLink {
+  path: string;
+  label: string;
+  match?: RegExp;
+}
+
+export interface IMenuGroup {
+  label: string;
+  links: IMenuLink[];
+}
+
+export interface IMenuAction {
+  icon?: string;
+  label?: string;
+  size?: "small" | "large" | "extra small" | "medium";
+  isActive?: boolean;
+  click?: () => void;
+}
+
+export interface ITenant extends IEntity, IName, ITimed, IContext {
+  systemscontext: string; //Which system the tenant record relates to
+  lat: string;
+  lon: string;
+  underlay: string;
+  epsg: string;
+}
+
+export interface IFileSelection {
+  id: string;
+  name: string;
+  datasource: any;
+}
+
+//This is a table of data in JSON format
+export type IDatasetTable = any[];
+
+//This type is used in the right type of the ICleanTransformCallback function and represents a set of named tables
+export type IDatasetsPackage = { [datatype: string]: IDatasetTable };
+
+//This type if the callback signature for the cleantransform callback. A dataset table and datatype are provided. A datasets package is returned
+export type ICleanTransformCallback = (
+  data: IDatasetTable,
+  datatype: string,
+) => IDatasetsPackage;
+
+//This callback allows filtering of data types
+export type IFilterCallback = (
+  datatype: string,
+  row: any,
+  links: IDocumentEntityLink[],
+) => boolean;
+
+// Used to derive primary keys for data
+export type IGetIdsForDataCallback = (
+  datatype: string,
+  data: IDatasetTable,
+) => string[];
+
+// Used in the update to get existing data by id which is then merged with the incoming updates
+export type IGetDataByIdsCallback = (
+  datatype: string,
+  ids: string[],
+) => Promise<IDatasetTable>;
+
+// Used in the update to combine existing and incoming updates to provide updates ready to overwrite existing data.
+export type ICombineExistingAndUpdateCallback = (
+  datatype: string,
+  updatedata: IDatasetTable,
+  existing: IDatasetTable,
+) => Promise<IDatasetTable>;
+
+// Used in the update to combine existing and incoming updates to provide updates ready to overwrite existing data.
+export type IFileChoiceCallback = () => IFileSelection[];