@dpuse/dpuse-shared 1.0.152 → 1.0.154

package/dist/types/src/component/module/connector/index.d.ts

@@ -3,7 +3,7 @@ import { Component } from '../..';
 import { EngineConnectorActionOptions } from '../engine';
 import { LocalisedConfig } from '../../../locale';
 import { ToolConfig } from '../tool';
-import { ConnectionDescriptionConfig, ConnectionNodeConfig, ObjectColumnConfig } from '../../connection';
+import { ConnectionConfig, ConnectionDescriptionConfig, ConnectionNodeConfig, ObjectColumnConfig } from '../../connection';
 import { connectorActionNameSchema, connectorCategoryConfigSchema, connectorConfigSchema, connectorItemTypeConfigSchema, connectorItemTypeIdSchema, connectorUsageConfigSchema, connectorUsageIdSchema } from './connectorConfig.schema';
 import { ContentAuditConfig, DataFormatId, InferenceRecord, InferenceSummary, ParsingRecord, PreviewConfig, ValueDelimiterId } from '../../dataView';
 export { connectorConfigSchema } from './connectorConfig.schema';
@@ -13,7 +13,7 @@ export interface ConnectorInterface extends Component {
     readonly config: ConnectorConfig;
     readonly toolConfigs: ToolConfig[];
     abortOperation?(): void;
-    auditObjectContent?(options: AuditObjectContentOptions, chunk: (rowCount: number) => void): Promise<AuditObjectContentResult>;
+    auditObjectContent?(options: AuditObjectContentOptions, chunk: (rowCount: number) => void, records?: (inferenceRecords: InferenceRecord[]) => Promise<void>): Promise<AuditObjectContentResult>;
     createObject?(options: CreateObjectOptions): Promise<void>;
     describeConnection?(options: DescribeConnectionOptions): Promise<DescribeConnectionResult>;
     dropObject?(options: DropObjectOptions): Promise<void>;
@@ -37,7 +37,7 @@ export interface ConnectorUtilities {
     inferDataTypes: (parsedRecords: ParsingRecord[]) => InferenceSummary;
 }
 export interface ColumnAuditor {
-    auditRecords: (parsedRecords: ParsingRecord[]) => void;
+    auditRecords: (parsedRecords: ParsingRecord[]) => InferenceRecord[];
     completeAudit: () => ObjectColumnConfig[];
 }
 export type ConnectorConfig = InferOutput<typeof connectorConfigSchema>;
@@ -52,12 +52,18 @@ export interface AuditObjectContentOptions extends EngineConnectorActionOptions
     hasHeaders: boolean;
     parserId: string | undefined;
     path: string;
+    recordStore?: RecordStoreConfig | undefined;
     supportsTransferableStreams: boolean;
     valueDelimiterId: ValueDelimiterId;
 }
 export interface AuditObjectContentResult {
     contentAuditConfig: ContentAuditConfig;
 }
+export interface RecordStoreConfig {
+    connectionConfig: ConnectionConfig;
+    nodeId: string;
+    storeId: string;
+}
 export interface CreateObjectOptions extends EngineConnectorActionOptions {
     path: string;
     structure: string;
@@ -130,8 +136,13 @@ export interface RetrieveRecordsOptions extends EngineConnectorActionOptions {
     encodingId: string;
     parserId: string | undefined;
     path: string;
+    range?: RecordRange | undefined;
     valueDelimiterId: ValueDelimiterId;
 }
+export interface RecordRange {
+    limit: number;
+    offset: number;
+}
 export interface RetrieveRecordsSummary {
     byteCount: number;
     commentLineCount: number;
@@ -139,6 +150,7 @@ export interface RetrieveRecordsSummary {
     lineCount: number;
     nonUniformRecordCount: number;
     recordCount: number;
+    totalCount?: number | undefined;
 }
 export type RecordRetrievalTypeId = 'jsonRecordArray' | 'parsingRecordArray';
 export interface UpsertRecordsOptions extends EngineConnectorActionOptions {

package/dist/types/src/component/module/connector/objectParsing.d.ts

@@ -1,6 +1,6 @@
 import { FileTypeResult } from 'file-type';
 import { AuditObjectContentOptions, ColumnAuditor, ConnectorUtilities, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
-import { ContentAuditConfig, DataFormatId, ParsingRecord, PreviewConfig } from '../../dataView';
+import { ContentAuditConfig, DataFormatId, InferenceRecord, ParsingRecord, PreviewConfig } from '../../dataView';
 import { ToolConfig } from '../tool';
 export interface ObjectFilePreview {
     bytes: Uint8Array;
@@ -29,6 +29,7 @@ export declare function retrieveRecordsWithParser(input: {
 export declare function auditObjectContentWithParser(input: {
     constructColumnAuditor: (hasHeaders: boolean) => ColumnAuditor;
     onProgress: (recordCount: number) => void;
+    onRecords?: ((inferenceRecords: InferenceRecord[]) => Promise<void>) | undefined;
     options: AuditObjectContentOptions;
     signal: AbortSignal;
     stream: ReadableStream<Uint8Array>;

package/dist/types/src/component/module/tool/parser.d.ts

@@ -1,7 +1,7 @@
 import { DataFormatId, ParsingRecord, PreviewIssue, RecordDelimiterId, ValueDelimiterId } from '../../dataView';
 import { RecordRetrievalTypeId, RetrieveRecordsSummary } from '../connector';
 export interface ParserInterface {
-    parseStream: (stream: ReadableStream<Uint8Array>, options: ParseStreamOptions, chunk: (typeId: RecordRetrievalTypeId, records: ParsingRecord[]) => void) => Promise<RetrieveRecordsSummary>;
+    parseStream: (stream: ReadableStream<Uint8Array>, options: ParseStreamOptions, chunk: (typeId: RecordRetrievalTypeId, records: ParsingRecord[]) => void | Promise<void>) => Promise<RetrieveRecordsSummary>;
     parseText: (text: string, options: ParseTextOptions) => Promise<ParseTextResult>;
 }
 export interface ParseTextOptions {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@dpuse/dpuse-shared",
-    "version": "1.0.152",
+    "version": "1.0.154",
     "private": false,
     "description": "Common constants, types and utilities used across DPUse projects.",
     "license": "MIT",
@@ -69,7 +69,7 @@
         "@types/node": "^26.6.5",
         "@vitest/coverage-v8": "^5.0.5",
         "eslint": "^10.13.0",
-        "fallow": "^3.36.0",
+        "fallow": "^3.37.0",
         "fast-check": "^4.10.2",
         "jschardet": "^4.1.0",
         "prettier": "^3.9.10",