@dpuse/dpuse-shared 1.0.155 → 1.0.159

package/dist/types/src/component/module/connector/connectorConfig.schema.d.ts

@@ -2,7 +2,7 @@
  * Defines the configuration metadata for a connector. Used for validation
  * of connector manifests and capability discovery at runtime.
  */
-export declare const connectorActionNameSchema: import('valibot').UnionSchema<readonly [import('valibot').LiteralSchema<"abortOperation", undefined>, import('valibot').LiteralSchema<"auditObjectContent", undefined>, import('valibot').LiteralSchema<"createObject", undefined>, import('valibot').LiteralSchema<"describeConnection", undefined>, import('valibot').LiteralSchema<"dropObject", undefined>, import('valibot').LiteralSchema<"findObject", undefined>, import('valibot').LiteralSchema<"getInfo", undefined>, import('valibot').LiteralSchema<"getReadableStream", undefined>, import('valibot').LiteralSchema<"getRecord", undefined>, import('valibot').LiteralSchema<"listNodes", undefined>, import('valibot').LiteralSchema<"previewObject", undefined>, import('valibot').LiteralSchema<"removeRecords", undefined>, import('valibot').LiteralSchema<"retrieveChunks", undefined>, import('valibot').LiteralSchema<"retrieveRecords", undefined>, import('valibot').LiteralSchema<"upsertRecords", undefined>], undefined>;
+export declare const connectorActionNameSchema: import('valibot').UnionSchema<readonly [import('valibot').LiteralSchema<"abortOperation", undefined>, import('valibot').LiteralSchema<"auditObjectContent", undefined>, import('valibot').LiteralSchema<"closeStore", undefined>, import('valibot').LiteralSchema<"createObject", undefined>, import('valibot').LiteralSchema<"describeConnection", undefined>, import('valibot').LiteralSchema<"dropObject", undefined>, import('valibot').LiteralSchema<"findObject", undefined>, import('valibot').LiteralSchema<"getInfo", undefined>, import('valibot').LiteralSchema<"getReadableStream", undefined>, import('valibot').LiteralSchema<"getRecord", undefined>, import('valibot').LiteralSchema<"listNodes", undefined>, import('valibot').LiteralSchema<"loadObjectRecords", undefined>, import('valibot').LiteralSchema<"openStore", undefined>, import('valibot').LiteralSchema<"previewObject", undefined>, import('valibot').LiteralSchema<"queryRecords", undefined>, import('valibot').LiteralSchema<"removeRecords", undefined>, import('valibot').LiteralSchema<"retrieveChunks", undefined>, import('valibot').LiteralSchema<"retrieveRecords", undefined>, import('valibot').LiteralSchema<"upsertRecords", undefined>], undefined>;
 export declare const connectorCategoryConfigSchema: import('valibot').StrictObjectSchema<{
     readonly label: import('valibot').StrictObjectSchema<{
         readonly en: import('valibot').StringSchema<undefined>;
@@ -25,7 +25,7 @@ export declare const connectorUsageConfigSchema: import('valibot').StrictObjectS
 }, undefined>;
 export declare const connectorConfigSchema: import('valibot').StrictObjectSchema<{
     readonly typeId: import('valibot').LiteralSchema<"connector", undefined>;
-    readonly actionNames: import('valibot').ArraySchema<import('valibot').UnionSchema<readonly [import('valibot').LiteralSchema<"abortOperation", undefined>, import('valibot').LiteralSchema<"auditObjectContent", undefined>, import('valibot').LiteralSchema<"createObject", undefined>, import('valibot').LiteralSchema<"describeConnection", undefined>, import('valibot').LiteralSchema<"dropObject", undefined>, import('valibot').LiteralSchema<"findObject", undefined>, import('valibot').LiteralSchema<"getInfo", undefined>, import('valibot').LiteralSchema<"getReadableStream", undefined>, import('valibot').LiteralSchema<"getRecord", undefined>, import('valibot').LiteralSchema<"listNodes", undefined>, import('valibot').LiteralSchema<"previewObject", undefined>, import('valibot').LiteralSchema<"removeRecords", undefined>, import('valibot').LiteralSchema<"retrieveChunks", undefined>, import('valibot').LiteralSchema<"retrieveRecords", undefined>, import('valibot').LiteralSchema<"upsertRecords", undefined>], undefined>, undefined>;
+    readonly actionNames: import('valibot').ArraySchema<import('valibot').UnionSchema<readonly [import('valibot').LiteralSchema<"abortOperation", undefined>, import('valibot').LiteralSchema<"auditObjectContent", undefined>, import('valibot').LiteralSchema<"closeStore", undefined>, import('valibot').LiteralSchema<"createObject", undefined>, import('valibot').LiteralSchema<"describeConnection", undefined>, import('valibot').LiteralSchema<"dropObject", undefined>, import('valibot').LiteralSchema<"findObject", undefined>, import('valibot').LiteralSchema<"getInfo", undefined>, import('valibot').LiteralSchema<"getReadableStream", undefined>, import('valibot').LiteralSchema<"getRecord", undefined>, import('valibot').LiteralSchema<"listNodes", undefined>, import('valibot').LiteralSchema<"loadObjectRecords", undefined>, import('valibot').LiteralSchema<"openStore", undefined>, import('valibot').LiteralSchema<"previewObject", undefined>, import('valibot').LiteralSchema<"queryRecords", undefined>, import('valibot').LiteralSchema<"removeRecords", undefined>, import('valibot').LiteralSchema<"retrieveChunks", undefined>, import('valibot').LiteralSchema<"retrieveRecords", undefined>, import('valibot').LiteralSchema<"upsertRecords", undefined>], undefined>, undefined>;
     readonly category: import('valibot').NullableSchema<import('valibot').StrictObjectSchema<{
         readonly label: import('valibot').StrictObjectSchema<{
             readonly en: import('valibot').StringSchema<undefined>;

package/dist/types/src/component/module/connector/index.d.ts

@@ -3,7 +3,7 @@ import { Component } from '../..';
 import { EngineConnectorActionOptions } from '../engine';
 import { LocalisedConfig } from '../../../locale';
 import { ToolConfig } from '../tool';
-import { ConnectionConfig, ConnectionDescriptionConfig, ConnectionNodeConfig, ObjectColumnConfig } from '../../connection';
+import { ConnectionDescriptionConfig, ConnectionNodeConfig, ObjectColumnConfig } from '../../connection';
 import { connectorActionNameSchema, connectorCategoryConfigSchema, connectorConfigSchema, connectorItemTypeConfigSchema, connectorItemTypeIdSchema, connectorUsageConfigSchema, connectorUsageIdSchema } from './connectorConfig.schema';
 import { ContentAuditConfig, DataFormatId, InferenceRecord, InferenceSummary, ParsingRecord, PreviewConfig, ValueDelimiterId } from '../../dataView';
 export { connectorConfigSchema } from './connectorConfig.schema';
@@ -13,7 +13,8 @@ export interface ConnectorInterface extends Component {
     readonly config: ConnectorConfig;
     readonly toolConfigs: ToolConfig[];
     abortOperation?(): void;
-    auditObjectContent?(options: AuditObjectContentOptions, chunk: (rowCount: number) => void, records?: (inferenceRecords: InferenceRecord[]) => Promise<void>): Promise<AuditObjectContentResult>;
+    auditObjectContent?(options: AuditObjectContentOptions, chunk: (rowCount: number) => void): Promise<AuditObjectContentResult>;
+    closeStore?(options: CloseStoreOptions): Promise<void>;
     createObject?(options: CreateObjectOptions): Promise<void>;
     describeConnection?(options: DescribeConnectionOptions): Promise<DescribeConnectionResult>;
     dropObject?(options: DropObjectOptions): Promise<void>;
@@ -22,7 +23,10 @@ export interface ConnectorInterface extends Component {
     getReadableStream?(options: GetReadableStreamOptions): Promise<ReadableStream<Uint8Array>>;
     getRecord?(options: GetRecordOptions): Promise<GetRecordResult>;
     listNodes?(options: ListNodesOptions): Promise<ListNodesResult>;
+    loadObjectRecords?(options: LoadObjectRecordsOptions, chunks: AsyncIterable<StoreRecord[]>): Promise<LoadObjectRecordsResult>;
+    openStore?(options: OpenStoreOptions): Promise<OpenStoreResult>;
     previewObject?(options: PreviewObjectOptions): Promise<PreviewConfig>;
+    queryRecords?(options: QueryRecordsOptions): Promise<QueryRecordsResult>;
     removeRecords?(options: RemoveRecordsOptions): Promise<void>;
     retrieveChunks?(options: RetrieveChunksOptions, chunk: (data: Uint8Array) => void, complete: () => void): Promise<void>;
     retrieveRecords?(options: RetrieveRecordsOptions, chunk: (typeId: RecordRetrievalTypeId, records: Record<string, unknown>[] | ParsingRecord[]) => void, complete: (result: RetrieveRecordsSummary) => void): Promise<void>;
@@ -52,18 +56,13 @@ export interface AuditObjectContentOptions extends EngineConnectorActionOptions
     hasHeaders: boolean;
     parserId: string | undefined;
     path: string;
-    recordStore?: RecordStoreConfig | undefined;
     supportsTransferableStreams: boolean;
     valueDelimiterId: ValueDelimiterId;
 }
 export interface AuditObjectContentResult {
     contentAuditConfig: ContentAuditConfig;
 }
-export interface RecordStoreConfig {
-    connectionConfig: ConnectionConfig;
-    nodeId: string;
-    storeId: string;
-}
+export type CloseStoreOptions = EngineConnectorActionOptions;
 export interface CreateObjectOptions extends EngineConnectorActionOptions {
     path: string;
     structure: string;
@@ -111,6 +110,20 @@ export interface ListNodesResult {
     isMore: boolean;
     totalCount: number;
 }
+export interface LoadObjectRecordsOptions extends EngineConnectorActionOptions {
+    columnConfigs: ObjectColumnConfig[];
+    path: string;
+}
+export interface LoadObjectRecordsResult {
+    recordCount: number;
+}
+export type StoreRecord = StoreValue[];
+export type StoreValue = bigint | boolean | Date | number | string | null;
+export type OpenStoreOptions = EngineConnectorActionOptions;
+export interface OpenStoreResult {
+    storeStateId: StoreStateId;
+}
+export type StoreStateId = 'heldElsewhere' | 'ready' | 'unavailable';
 export interface PreviewObjectOptions extends EngineConnectorActionOptions {
     chunkSize: number | undefined;
     dataFormatId: DataFormatId | undefined;
@@ -120,6 +133,34 @@ export interface PreviewObjectOptions extends EngineConnectorActionOptions {
     parserId: string | undefined;
     path: string;
 }
+export interface QueryRecordsOptions extends EngineConnectorActionOptions {
+    aggregates: QueryAggregate[];
+    filters: QueryFilter[];
+    groupColumnIndexes: number[];
+    limit: number;
+    offset: number;
+    path: string;
+    sorts: QuerySort[];
+}
+export interface QueryAggregate {
+    columnIndex: number | undefined;
+    functionId: QueryAggregateFunctionId;
+}
+export type QueryAggregateFunctionId = 'average' | 'count' | 'countDistinct' | 'maximum' | 'minimum' | 'sum';
+export interface QueryFilter {
+    columnIndex: number;
+    operatorId: QueryFilterOperatorId;
+    value: StoreValue;
+}
+export type QueryFilterOperatorId = 'contains' | 'endsWith' | 'equals' | 'greaterThan' | 'greaterThanOrEquals' | 'isEmpty' | 'isNotEmpty' | 'lessThan' | 'lessThanOrEquals' | 'notEquals' | 'startsWith';
+export interface QuerySort {
+    columnIndex: number;
+    directionId: 'ascending' | 'descending';
+}
+export interface QueryRecordsResult {
+    records: StoreRecord[];
+    totalCount: number;
+}
 export interface RemoveRecordsOptions extends EngineConnectorActionOptions {
     keys: string[];
     path: string;

package/dist/types/src/component/module/connector/objectParsing.d.ts

@@ -1,6 +1,6 @@
 import { FileTypeResult } from 'file-type';
 import { AuditObjectContentOptions, ColumnAuditor, ConnectorUtilities, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
-import { ContentAuditConfig, DataFormatId, InferenceRecord, ParsingRecord, PreviewConfig } from '../../dataView';
+import { ContentAuditConfig, DataFormatId, ParsingRecord, PreviewConfig } from '../../dataView';
 import { ToolConfig } from '../tool';
 export interface ObjectFilePreview {
     bytes: Uint8Array;
@@ -29,7 +29,6 @@ export declare function retrieveRecordsWithParser(input: {
 export declare function auditObjectContentWithParser(input: {
     constructColumnAuditor: (hasHeaders: boolean) => ColumnAuditor;
     onProgress: (recordCount: number) => void;
-    onRecords?: ((inferenceRecords: InferenceRecord[]) => Promise<void>) | undefined;
     options: AuditObjectContentOptions;
     signal: AbortSignal;
     stream: ReadableStream<Uint8Array>;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@dpuse/dpuse-shared",
-    "version": "1.0.155",
+    "version": "1.0.159",
     "private": false,
     "description": "Common constants, types and utilities used across DPUse projects.",
     "license": "MIT",
@@ -64,7 +64,7 @@
         "valibot": "^1.5.0"
     },
     "devDependencies": {
-        "@dpuse/dpuse-development": "^1.0.135",
+        "@dpuse/dpuse-development": "^1.0.140",
         "@dpuse/eslint-config-dpuse": "^1.0.145",
         "@types/node": "^26.6.5",
         "@vitest/coverage-v8": "^5.0.5",