@dpuse/dpuse-shared 1.0.170 → 1.0.171

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -5,51 +5,48 @@ export interface DataViewConfig extends ComponentInstanceConfig {
5
5
  typeId: 'dataView';
6
6
  connectionId: string;
7
7
  connectionNodeConfig: ConnectionNodeConfig | undefined;
8
- previewConfig: StoredPreviewConfig | undefined;
8
+ readConfig: ReadConfig | undefined;
9
+ previewConfig: PreviewConfig | undefined;
10
+ contentAuditConfig: StoredContentAuditConfig | undefined;
9
11
  columnSetConfig: ColumnSetConfig | undefined;
10
- auditSummaryConfig: AuditSummaryConfig | undefined;
11
12
  relationshipsAuditConfig: RelationshipsAuditConfig | undefined;
12
13
  }
13
14
  export interface ColumnSetConfig {
14
15
  asAt: number;
15
16
  columnConfigs: DataViewColumnConfig[];
16
- sourceId: ColumnSetSourceId;
17
17
  }
18
- export type ColumnSetSourceId = 'audit' | 'preview';
19
18
  export interface DataViewColumnConfig extends ObjectColumnConfig {
20
19
  chosenDataTypeId?: DataTypeId | undefined;
21
20
  }
22
- export interface PreviewConfig {
23
- asAt: number;
21
+ export interface ReadConfig {
24
22
  commentMarkId?: string | undefined;
25
23
  commentMarkOtherCharSeq?: string | undefined;
26
- columnConfigs: ObjectColumnConfig[];
27
24
  dataFormatId: DataFormatId;
28
- duration: number;
29
25
  encodingConfidenceLevel: number | undefined;
30
26
  encodingId: string | undefined;
31
- errorMessage?: string | undefined;
32
27
  fileType: FileTypeResult | undefined;
33
28
  hasHeaders: boolean | undefined;
34
- issues?: PreviewIssue[];
35
29
  linesToSkipAtStart?: number | undefined;
36
- parsedRecords: ParsingRecord[];
37
30
  parserId?: string | undefined;
38
31
  quoteEscapeChar?: string | undefined;
39
32
  quoteMarkId?: string | undefined;
40
33
  quoteMarkOtherCharSeq?: string | undefined;
41
34
  recordDelimiterId: RecordDelimiterId | undefined;
42
35
  recordDelimiterOtherCharSeq?: string | undefined;
43
- size: number | undefined;
44
36
  skipEmptyLines?: boolean | undefined;
45
37
  skipLinesWithEmptyValues?: boolean | undefined;
46
38
  skipLinesWithErrors?: boolean | undefined;
47
- text: string | undefined;
48
39
  valueDelimiterId: ValueDelimiterId | undefined;
49
40
  valueDelimiterOtherCharSeq?: string | undefined;
50
41
  valueTrimMethodId?: ValueTrimMethodId | undefined;
51
42
  }
52
- export type StoredPreviewConfig = Omit<PreviewConfig, 'columnConfigs'>;
43
+ export interface PreviewConfig {
44
+ errorMessage?: string | undefined;
45
+ issues?: PreviewIssue[];
46
+ parsedRecords: ParsingRecord[];
47
+ size: number | undefined;
48
+ text: string | undefined;
49
+ }
53
50
  export interface PreviewIssue {
54
51
  endLineNumber?: number | undefined;
55
52
  fieldNumber?: number | undefined;
@@ -72,16 +69,18 @@ export type ValueDelimiterId = '' | ':' | ',' | '!' | '0x1E' | ';' | ' ' | '\t'
72
69
  type ValueTrimMethodId = 'both' | 'left' | 'right' | 'none';
73
70
  export interface ContentAuditConfig {
74
71
  asAt: number;
72
+ byteCount: number;
75
73
  columns: ObjectColumnConfig[];
76
74
  commentLineCount: number | undefined;
77
75
  emptyLineCount: number | undefined;
78
76
  invalidFieldLengthCount: number | undefined;
79
77
  duration: number;
78
+ isComplete: boolean;
80
79
  lineCount: number | undefined;
81
80
  recordCount: number;
82
81
  sourceWaitDuration: number | undefined;
83
82
  }
84
- export type AuditSummaryConfig = Omit<ContentAuditConfig, 'columns'>;
83
+ export type StoredContentAuditConfig = Omit<ContentAuditConfig, 'columns'>;
85
84
  interface RelationshipsAuditConfig {
86
85
  placeholder: string;
87
86
  }
@@ -5,15 +5,15 @@ import { LocalisedConfig } from '../../../locale';
5
5
  import { ToolConfig } from '../tool';
6
6
  import { ConnectionConfig, ConnectionDescriptionConfig, ConnectionNodeConfig, ObjectColumnConfig } from '../../connection';
7
7
  import { connectorActionNameSchema, connectorCategoryConfigSchema, connectorConfigSchema, connectorItemTypeConfigSchema, connectorItemTypeIdSchema, connectorUsageConfigSchema, connectorUsageIdSchema } from './connectorConfig.schema';
8
- import { ContentAuditConfig, DataFormatId, InferenceRecord, InferenceSummary, ParsingRecord, PreviewConfig, ValueDelimiterId } from '../../dataView';
8
+ import { ContentAuditConfig, DataFormatId, InferenceRecord, InferenceSummary, ParsingRecord, PreviewConfig, ReadConfig, ValueDelimiterId } from '../../dataView';
9
9
  export { connectorConfigSchema } from './connectorConfig.schema';
10
- export { auditObjectContentWithParser, buildPreviewConfig, type ObjectFilePreview, retrieveRecordsWithParser } from './objectParsing';
10
+ export { auditObjectContentWithParser, buildPreviewObjectResult, type ObjectFilePreview, retrieveRecordsWithParser } from './objectParsing';
11
11
  export interface ConnectorInterface extends Component {
12
12
  abortController: AbortController | undefined;
13
13
  readonly config: ConnectorConfig;
14
14
  readonly toolConfigs: ToolConfig[];
15
15
  abortOperation?(): void;
16
- auditObjectContent?(options: AuditObjectContentOptions, chunk: (rowCount: number) => void): Promise<AuditObjectContentResult>;
16
+ auditObjectContent?(options: AuditObjectContentOptions, chunk: (recordCount: number, byteCount: number) => void): Promise<AuditObjectContentResult>;
17
17
  closeStore?(options: CloseStoreOptions): Promise<void>;
18
18
  createObject?(options: CreateObjectOptions): Promise<void>;
19
19
  describeConnection?(options: DescribeConnectionOptions): Promise<DescribeConnectionResult>;
@@ -25,7 +25,7 @@ export interface ConnectorInterface extends Component {
25
25
  listNodes?(options: ListNodesOptions): Promise<ListNodesResult>;
26
26
  loadObjectRecords?(options: LoadObjectRecordsOptions, chunks: AsyncIterable<StoreRecord[]>): Promise<LoadObjectRecordsResult>;
27
27
  openStore?(options: OpenStoreOptions): Promise<OpenStoreResult>;
28
- previewObject?(options: PreviewObjectOptions): Promise<PreviewConfig>;
28
+ previewObject?(options: PreviewObjectOptions): Promise<PreviewObjectResult>;
29
29
  queryRecords?(options: QueryRecordsOptions): Promise<QueryRecordsResult>;
30
30
  removeRecords?(options: RemoveRecordsOptions): Promise<void>;
31
31
  retrieveChunks?(options: RetrieveChunksOptions, chunk: (data: Uint8Array) => void, complete: () => void): Promise<void>;
@@ -51,12 +51,14 @@ type ConnectorItemTypeConfig = InferOutput<typeof connectorItemTypeConfigSchema>
51
51
  export type ConnectorItemTypeId = InferOutput<typeof connectorItemTypeIdSchema>;
52
52
  type ConnectorUsageConfig = InferOutput<typeof connectorUsageConfigSchema>;
53
53
  export interface AuditObjectContentOptions extends EngineConnectorActionOptions {
54
+ byteLimit: number | undefined;
54
55
  chunkSize: number | undefined;
55
56
  dataFormatId: DataFormatId;
56
57
  encodingId: string;
57
58
  hasHeaders: boolean;
58
59
  parserId: string | undefined;
59
60
  path: string;
61
+ stopSignal: AbortSignal | undefined;
60
62
  supportsTransferableStreams: boolean;
61
63
  valueDelimiterId: ValueDelimiterId;
62
64
  }
@@ -145,6 +147,11 @@ export interface PreviewObjectOptions extends EngineConnectorActionOptions {
145
147
  path: string;
146
148
  valueDelimiterId: ValueDelimiterId | undefined;
147
149
  }
150
+ export interface PreviewObjectResult {
151
+ contentAuditConfig: ContentAuditConfig;
152
+ previewConfig: PreviewConfig;
153
+ readConfig: ReadConfig;
154
+ }
148
155
  export interface QueryRecordsOptions extends EngineConnectorActionOptions {
149
156
  aggregates: QueryAggregate[];
150
157
  filters: QueryFilter[];
@@ -1,6 +1,6 @@
1
1
  import { FileTypeResult } from 'file-type';
2
- import { AuditObjectContentOptions, ColumnAuditor, ConnectorUtilities, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
3
- import { ContentAuditConfig, DataFormatId, ParsingRecord, PreviewConfig, ValueDelimiterId } from '../../dataView';
2
+ import { AuditObjectContentOptions, ColumnAuditor, ConnectorUtilities, PreviewObjectResult, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
3
+ import { ContentAuditConfig, DataFormatId, ParsingRecord, ValueDelimiterId } from '../../dataView';
4
4
  import { ToolConfig } from '../tool';
5
5
  export interface ObjectFilePreview {
6
6
  bytes: Uint8Array;
@@ -11,7 +11,7 @@ export interface ObjectFilePreview {
11
11
  isComplete: boolean;
12
12
  text: string | undefined;
13
13
  }
14
- export declare function buildPreviewConfig(input: {
14
+ export declare function buildPreviewObjectResult(input: {
15
15
  asAt: number;
16
16
  filePreview: ObjectFilePreview;
17
17
  inferDataTypes: ConnectorUtilities['inferDataTypes'];
@@ -19,7 +19,7 @@ export declare function buildPreviewConfig(input: {
19
19
  startedAt: number;
20
20
  toolConfigs: ToolConfig[];
21
21
  valueDelimiterId: ValueDelimiterId | undefined;
22
- }): Promise<PreviewConfig>;
22
+ }): Promise<PreviewObjectResult>;
23
23
  export declare function retrieveRecordsWithParser(input: {
24
24
  chunk: (typeId: RecordRetrievalTypeId, records: ParsingRecord[]) => void | Promise<void>;
25
25
  options: RetrieveRecordsOptions;
@@ -29,7 +29,7 @@ export declare function retrieveRecordsWithParser(input: {
29
29
  }): Promise<RetrieveRecordsSummary>;
30
30
  export declare function auditObjectContentWithParser(input: {
31
31
  constructColumnAuditor: (hasHeaders: boolean) => ColumnAuditor;
32
- onProgress: (recordCount: number) => void;
32
+ onProgress: (recordCount: number, byteCount: number) => void;
33
33
  options: AuditObjectContentOptions;
34
34
  signal: AbortSignal;
35
35
  stream: ReadableStream<Uint8Array>;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dpuse/dpuse-shared",
3
- "version": "1.0.170",
3
+ "version": "1.0.171",
4
4
  "private": false,
5
5
  "description": "Common constants, types and utilities used across DPUse projects.",
6
6
  "license": "MIT",
@@ -64,8 +64,8 @@
64
64
  "valibot": "^1.5.0"
65
65
  },
66
66
  "devDependencies": {
67
- "@dpuse/dpuse-development": "^1.0.141",
68
- "@dpuse/eslint-config-dpuse": "^1.0.145",
67
+ "@dpuse/dpuse-development": "^1.0.144",
68
+ "@dpuse/eslint-config-dpuse": "^1.0.148",
69
69
  "@types/node": "^26.6.6",
70
70
  "@vitest/coverage-v8": "^5.0.5",
71
71
  "eslint": "^10.14.0",