@dpuse/dpuse-shared 1.0.169 → 1.0.171

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -5,40 +5,48 @@ export interface DataViewConfig extends ComponentInstanceConfig {
5
5
  typeId: 'dataView';
6
6
  connectionId: string;
7
7
  connectionNodeConfig: ConnectionNodeConfig | undefined;
8
+ readConfig: ReadConfig | undefined;
8
9
  previewConfig: PreviewConfig | undefined;
9
- contentAuditConfig: ContentAuditConfig | undefined;
10
+ contentAuditConfig: StoredContentAuditConfig | undefined;
11
+ columnSetConfig: ColumnSetConfig | undefined;
10
12
  relationshipsAuditConfig: RelationshipsAuditConfig | undefined;
11
13
  }
12
- export interface PreviewConfig {
14
+ export interface ColumnSetConfig {
13
15
  asAt: number;
16
+ columnConfigs: DataViewColumnConfig[];
17
+ }
18
+ export interface DataViewColumnConfig extends ObjectColumnConfig {
19
+ chosenDataTypeId?: DataTypeId | undefined;
20
+ }
21
+ export interface ReadConfig {
14
22
  commentMarkId?: string | undefined;
15
23
  commentMarkOtherCharSeq?: string | undefined;
16
- columnConfigs: ObjectColumnConfig[];
17
24
  dataFormatId: DataFormatId;
18
- duration: number;
19
25
  encodingConfidenceLevel: number | undefined;
20
26
  encodingId: string | undefined;
21
- errorMessage?: string | undefined;
22
27
  fileType: FileTypeResult | undefined;
23
28
  hasHeaders: boolean | undefined;
24
- issues?: PreviewIssue[];
25
29
  linesToSkipAtStart?: number | undefined;
26
- parsedRecords: ParsingRecord[];
27
30
  parserId?: string | undefined;
28
31
  quoteEscapeChar?: string | undefined;
29
32
  quoteMarkId?: string | undefined;
30
33
  quoteMarkOtherCharSeq?: string | undefined;
31
34
  recordDelimiterId: RecordDelimiterId | undefined;
32
35
  recordDelimiterOtherCharSeq?: string | undefined;
33
- size: number | undefined;
34
36
  skipEmptyLines?: boolean | undefined;
35
37
  skipLinesWithEmptyValues?: boolean | undefined;
36
38
  skipLinesWithErrors?: boolean | undefined;
37
- text: string | undefined;
38
39
  valueDelimiterId: ValueDelimiterId | undefined;
39
40
  valueDelimiterOtherCharSeq?: string | undefined;
40
41
  valueTrimMethodId?: ValueTrimMethodId | undefined;
41
42
  }
43
+ export interface PreviewConfig {
44
+ errorMessage?: string | undefined;
45
+ issues?: PreviewIssue[];
46
+ parsedRecords: ParsingRecord[];
47
+ size: number | undefined;
48
+ text: string | undefined;
49
+ }
42
50
  export interface PreviewIssue {
43
51
  endLineNumber?: number | undefined;
44
52
  fieldNumber?: number | undefined;
@@ -61,15 +69,18 @@ export type ValueDelimiterId = '' | ':' | ',' | '!' | '0x1E' | ';' | ' ' | '\t'
61
69
  type ValueTrimMethodId = 'both' | 'left' | 'right' | 'none';
62
70
  export interface ContentAuditConfig {
63
71
  asAt: number;
72
+ byteCount: number;
64
73
  columns: ObjectColumnConfig[];
65
74
  commentLineCount: number | undefined;
66
75
  emptyLineCount: number | undefined;
67
76
  invalidFieldLengthCount: number | undefined;
68
77
  duration: number;
78
+ isComplete: boolean;
69
79
  lineCount: number | undefined;
70
80
  recordCount: number;
71
81
  sourceWaitDuration: number | undefined;
72
82
  }
83
+ export type StoredContentAuditConfig = Omit<ContentAuditConfig, 'columns'>;
73
84
  interface RelationshipsAuditConfig {
74
85
  placeholder: string;
75
86
  }
@@ -5,15 +5,15 @@ import { LocalisedConfig } from '../../../locale';
5
5
  import { ToolConfig } from '../tool';
6
6
  import { ConnectionConfig, ConnectionDescriptionConfig, ConnectionNodeConfig, ObjectColumnConfig } from '../../connection';
7
7
  import { connectorActionNameSchema, connectorCategoryConfigSchema, connectorConfigSchema, connectorItemTypeConfigSchema, connectorItemTypeIdSchema, connectorUsageConfigSchema, connectorUsageIdSchema } from './connectorConfig.schema';
8
- import { ContentAuditConfig, DataFormatId, InferenceRecord, InferenceSummary, ParsingRecord, PreviewConfig, ValueDelimiterId } from '../../dataView';
8
+ import { ContentAuditConfig, DataFormatId, InferenceRecord, InferenceSummary, ParsingRecord, PreviewConfig, ReadConfig, ValueDelimiterId } from '../../dataView';
9
9
  export { connectorConfigSchema } from './connectorConfig.schema';
10
- export { auditObjectContentWithParser, buildPreviewConfig, type ObjectFilePreview, retrieveRecordsWithParser } from './objectParsing';
10
+ export { auditObjectContentWithParser, buildPreviewObjectResult, type ObjectFilePreview, retrieveRecordsWithParser } from './objectParsing';
11
11
  export interface ConnectorInterface extends Component {
12
12
  abortController: AbortController | undefined;
13
13
  readonly config: ConnectorConfig;
14
14
  readonly toolConfigs: ToolConfig[];
15
15
  abortOperation?(): void;
16
- auditObjectContent?(options: AuditObjectContentOptions, chunk: (rowCount: number) => void): Promise<AuditObjectContentResult>;
16
+ auditObjectContent?(options: AuditObjectContentOptions, chunk: (recordCount: number, byteCount: number) => void): Promise<AuditObjectContentResult>;
17
17
  closeStore?(options: CloseStoreOptions): Promise<void>;
18
18
  createObject?(options: CreateObjectOptions): Promise<void>;
19
19
  describeConnection?(options: DescribeConnectionOptions): Promise<DescribeConnectionResult>;
@@ -25,7 +25,7 @@ export interface ConnectorInterface extends Component {
25
25
  listNodes?(options: ListNodesOptions): Promise<ListNodesResult>;
26
26
  loadObjectRecords?(options: LoadObjectRecordsOptions, chunks: AsyncIterable<StoreRecord[]>): Promise<LoadObjectRecordsResult>;
27
27
  openStore?(options: OpenStoreOptions): Promise<OpenStoreResult>;
28
- previewObject?(options: PreviewObjectOptions): Promise<PreviewConfig>;
28
+ previewObject?(options: PreviewObjectOptions): Promise<PreviewObjectResult>;
29
29
  queryRecords?(options: QueryRecordsOptions): Promise<QueryRecordsResult>;
30
30
  removeRecords?(options: RemoveRecordsOptions): Promise<void>;
31
31
  retrieveChunks?(options: RetrieveChunksOptions, chunk: (data: Uint8Array) => void, complete: () => void): Promise<void>;
@@ -51,12 +51,14 @@ type ConnectorItemTypeConfig = InferOutput<typeof connectorItemTypeConfigSchema>
51
51
  export type ConnectorItemTypeId = InferOutput<typeof connectorItemTypeIdSchema>;
52
52
  type ConnectorUsageConfig = InferOutput<typeof connectorUsageConfigSchema>;
53
53
  export interface AuditObjectContentOptions extends EngineConnectorActionOptions {
54
+ byteLimit: number | undefined;
54
55
  chunkSize: number | undefined;
55
56
  dataFormatId: DataFormatId;
56
57
  encodingId: string;
57
58
  hasHeaders: boolean;
58
59
  parserId: string | undefined;
59
60
  path: string;
61
+ stopSignal: AbortSignal | undefined;
60
62
  supportsTransferableStreams: boolean;
61
63
  valueDelimiterId: ValueDelimiterId;
62
64
  }
@@ -120,9 +122,14 @@ export interface LoadObjectRecordsResult {
120
122
  }
121
123
  export interface EngineLoadObjectRecordsOptions extends LoadObjectRecordsOptions {
122
124
  hasHeaders: boolean;
125
+ headerValues: (string | null)[] | undefined;
123
126
  sourceConnectionConfig: ConnectionConfig | LocalisedConfig<ConnectionConfig>;
124
127
  sourceOptions: RetrieveRecordsOptions;
125
128
  }
129
+ export interface EngineLoadObjectRecordsResult extends LoadObjectRecordsResult {
130
+ columnsHaveChanged: boolean;
131
+ invalidValueCounts: number[];
132
+ }
126
133
  export type StoreRecord = StoreValue[];
127
134
  export type StoreValue = bigint | boolean | Date | number | string | null;
128
135
  export type OpenStoreOptions = EngineConnectorActionOptions;
@@ -140,6 +147,11 @@ export interface PreviewObjectOptions extends EngineConnectorActionOptions {
140
147
  path: string;
141
148
  valueDelimiterId: ValueDelimiterId | undefined;
142
149
  }
150
+ export interface PreviewObjectResult {
151
+ contentAuditConfig: ContentAuditConfig;
152
+ previewConfig: PreviewConfig;
153
+ readConfig: ReadConfig;
154
+ }
143
155
  export interface QueryRecordsOptions extends EngineConnectorActionOptions {
144
156
  aggregates: QueryAggregate[];
145
157
  filters: QueryFilter[];
@@ -1,6 +1,6 @@
1
1
  import { FileTypeResult } from 'file-type';
2
- import { AuditObjectContentOptions, ColumnAuditor, ConnectorUtilities, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
3
- import { ContentAuditConfig, DataFormatId, ParsingRecord, PreviewConfig, ValueDelimiterId } from '../../dataView';
2
+ import { AuditObjectContentOptions, ColumnAuditor, ConnectorUtilities, PreviewObjectResult, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
3
+ import { ContentAuditConfig, DataFormatId, ParsingRecord, ValueDelimiterId } from '../../dataView';
4
4
  import { ToolConfig } from '../tool';
5
5
  export interface ObjectFilePreview {
6
6
  bytes: Uint8Array;
@@ -11,7 +11,7 @@ export interface ObjectFilePreview {
11
11
  isComplete: boolean;
12
12
  text: string | undefined;
13
13
  }
14
- export declare function buildPreviewConfig(input: {
14
+ export declare function buildPreviewObjectResult(input: {
15
15
  asAt: number;
16
16
  filePreview: ObjectFilePreview;
17
17
  inferDataTypes: ConnectorUtilities['inferDataTypes'];
@@ -19,7 +19,7 @@ export declare function buildPreviewConfig(input: {
19
19
  startedAt: number;
20
20
  toolConfigs: ToolConfig[];
21
21
  valueDelimiterId: ValueDelimiterId | undefined;
22
- }): Promise<PreviewConfig>;
22
+ }): Promise<PreviewObjectResult>;
23
23
  export declare function retrieveRecordsWithParser(input: {
24
24
  chunk: (typeId: RecordRetrievalTypeId, records: ParsingRecord[]) => void | Promise<void>;
25
25
  options: RetrieveRecordsOptions;
@@ -29,7 +29,7 @@ export declare function retrieveRecordsWithParser(input: {
29
29
  }): Promise<RetrieveRecordsSummary>;
30
30
  export declare function auditObjectContentWithParser(input: {
31
31
  constructColumnAuditor: (hasHeaders: boolean) => ColumnAuditor;
32
- onProgress: (recordCount: number) => void;
32
+ onProgress: (recordCount: number, byteCount: number) => void;
33
33
  options: AuditObjectContentOptions;
34
34
  signal: AbortSignal;
35
35
  stream: ReadableStream<Uint8Array>;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dpuse/dpuse-shared",
3
- "version": "1.0.169",
3
+ "version": "1.0.171",
4
4
  "private": false,
5
5
  "description": "Common constants, types and utilities used across DPUse projects.",
6
6
  "license": "MIT",
@@ -64,8 +64,8 @@
64
64
  "valibot": "^1.5.0"
65
65
  },
66
66
  "devDependencies": {
67
- "@dpuse/dpuse-development": "^1.0.141",
68
- "@dpuse/eslint-config-dpuse": "^1.0.145",
67
+ "@dpuse/dpuse-development": "^1.0.144",
68
+ "@dpuse/eslint-config-dpuse": "^1.0.148",
69
69
  "@types/node": "^26.6.6",
70
70
  "@vitest/coverage-v8": "^5.0.5",
71
71
  "eslint": "^10.14.0",