@dpuse/dpuse-shared 1.0.169 → 1.0.171
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -9
- package/dist/dpuse-shared.es.js +783 -705
- package/dist/dpuse-shared.es.js.map +1 -1
- package/dist/types/src/component/dataView/index.d.ts +20 -9
- package/dist/types/src/component/module/connector/index.d.ts +16 -4
- package/dist/types/src/component/module/connector/objectParsing.d.ts +5 -5
- package/package.json +3 -3
|
@@ -5,40 +5,48 @@ export interface DataViewConfig extends ComponentInstanceConfig {
|
|
|
5
5
|
typeId: 'dataView';
|
|
6
6
|
connectionId: string;
|
|
7
7
|
connectionNodeConfig: ConnectionNodeConfig | undefined;
|
|
8
|
+
readConfig: ReadConfig | undefined;
|
|
8
9
|
previewConfig: PreviewConfig | undefined;
|
|
9
|
-
contentAuditConfig:
|
|
10
|
+
contentAuditConfig: StoredContentAuditConfig | undefined;
|
|
11
|
+
columnSetConfig: ColumnSetConfig | undefined;
|
|
10
12
|
relationshipsAuditConfig: RelationshipsAuditConfig | undefined;
|
|
11
13
|
}
|
|
12
|
-
export interface
|
|
14
|
+
export interface ColumnSetConfig {
|
|
13
15
|
asAt: number;
|
|
16
|
+
columnConfigs: DataViewColumnConfig[];
|
|
17
|
+
}
|
|
18
|
+
export interface DataViewColumnConfig extends ObjectColumnConfig {
|
|
19
|
+
chosenDataTypeId?: DataTypeId | undefined;
|
|
20
|
+
}
|
|
21
|
+
export interface ReadConfig {
|
|
14
22
|
commentMarkId?: string | undefined;
|
|
15
23
|
commentMarkOtherCharSeq?: string | undefined;
|
|
16
|
-
columnConfigs: ObjectColumnConfig[];
|
|
17
24
|
dataFormatId: DataFormatId;
|
|
18
|
-
duration: number;
|
|
19
25
|
encodingConfidenceLevel: number | undefined;
|
|
20
26
|
encodingId: string | undefined;
|
|
21
|
-
errorMessage?: string | undefined;
|
|
22
27
|
fileType: FileTypeResult | undefined;
|
|
23
28
|
hasHeaders: boolean | undefined;
|
|
24
|
-
issues?: PreviewIssue[];
|
|
25
29
|
linesToSkipAtStart?: number | undefined;
|
|
26
|
-
parsedRecords: ParsingRecord[];
|
|
27
30
|
parserId?: string | undefined;
|
|
28
31
|
quoteEscapeChar?: string | undefined;
|
|
29
32
|
quoteMarkId?: string | undefined;
|
|
30
33
|
quoteMarkOtherCharSeq?: string | undefined;
|
|
31
34
|
recordDelimiterId: RecordDelimiterId | undefined;
|
|
32
35
|
recordDelimiterOtherCharSeq?: string | undefined;
|
|
33
|
-
size: number | undefined;
|
|
34
36
|
skipEmptyLines?: boolean | undefined;
|
|
35
37
|
skipLinesWithEmptyValues?: boolean | undefined;
|
|
36
38
|
skipLinesWithErrors?: boolean | undefined;
|
|
37
|
-
text: string | undefined;
|
|
38
39
|
valueDelimiterId: ValueDelimiterId | undefined;
|
|
39
40
|
valueDelimiterOtherCharSeq?: string | undefined;
|
|
40
41
|
valueTrimMethodId?: ValueTrimMethodId | undefined;
|
|
41
42
|
}
|
|
43
|
+
export interface PreviewConfig {
|
|
44
|
+
errorMessage?: string | undefined;
|
|
45
|
+
issues?: PreviewIssue[];
|
|
46
|
+
parsedRecords: ParsingRecord[];
|
|
47
|
+
size: number | undefined;
|
|
48
|
+
text: string | undefined;
|
|
49
|
+
}
|
|
42
50
|
export interface PreviewIssue {
|
|
43
51
|
endLineNumber?: number | undefined;
|
|
44
52
|
fieldNumber?: number | undefined;
|
|
@@ -61,15 +69,18 @@ export type ValueDelimiterId = '' | ':' | ',' | '!' | '0x1E' | ';' | ' ' | '\t'
|
|
|
61
69
|
type ValueTrimMethodId = 'both' | 'left' | 'right' | 'none';
|
|
62
70
|
export interface ContentAuditConfig {
|
|
63
71
|
asAt: number;
|
|
72
|
+
byteCount: number;
|
|
64
73
|
columns: ObjectColumnConfig[];
|
|
65
74
|
commentLineCount: number | undefined;
|
|
66
75
|
emptyLineCount: number | undefined;
|
|
67
76
|
invalidFieldLengthCount: number | undefined;
|
|
68
77
|
duration: number;
|
|
78
|
+
isComplete: boolean;
|
|
69
79
|
lineCount: number | undefined;
|
|
70
80
|
recordCount: number;
|
|
71
81
|
sourceWaitDuration: number | undefined;
|
|
72
82
|
}
|
|
83
|
+
export type StoredContentAuditConfig = Omit<ContentAuditConfig, 'columns'>;
|
|
73
84
|
interface RelationshipsAuditConfig {
|
|
74
85
|
placeholder: string;
|
|
75
86
|
}
|
|
@@ -5,15 +5,15 @@ import { LocalisedConfig } from '../../../locale';
|
|
|
5
5
|
import { ToolConfig } from '../tool';
|
|
6
6
|
import { ConnectionConfig, ConnectionDescriptionConfig, ConnectionNodeConfig, ObjectColumnConfig } from '../../connection';
|
|
7
7
|
import { connectorActionNameSchema, connectorCategoryConfigSchema, connectorConfigSchema, connectorItemTypeConfigSchema, connectorItemTypeIdSchema, connectorUsageConfigSchema, connectorUsageIdSchema } from './connectorConfig.schema';
|
|
8
|
-
import { ContentAuditConfig, DataFormatId, InferenceRecord, InferenceSummary, ParsingRecord, PreviewConfig, ValueDelimiterId } from '../../dataView';
|
|
8
|
+
import { ContentAuditConfig, DataFormatId, InferenceRecord, InferenceSummary, ParsingRecord, PreviewConfig, ReadConfig, ValueDelimiterId } from '../../dataView';
|
|
9
9
|
export { connectorConfigSchema } from './connectorConfig.schema';
|
|
10
|
-
export { auditObjectContentWithParser,
|
|
10
|
+
export { auditObjectContentWithParser, buildPreviewObjectResult, type ObjectFilePreview, retrieveRecordsWithParser } from './objectParsing';
|
|
11
11
|
export interface ConnectorInterface extends Component {
|
|
12
12
|
abortController: AbortController | undefined;
|
|
13
13
|
readonly config: ConnectorConfig;
|
|
14
14
|
readonly toolConfigs: ToolConfig[];
|
|
15
15
|
abortOperation?(): void;
|
|
16
|
-
auditObjectContent?(options: AuditObjectContentOptions, chunk: (
|
|
16
|
+
auditObjectContent?(options: AuditObjectContentOptions, chunk: (recordCount: number, byteCount: number) => void): Promise<AuditObjectContentResult>;
|
|
17
17
|
closeStore?(options: CloseStoreOptions): Promise<void>;
|
|
18
18
|
createObject?(options: CreateObjectOptions): Promise<void>;
|
|
19
19
|
describeConnection?(options: DescribeConnectionOptions): Promise<DescribeConnectionResult>;
|
|
@@ -25,7 +25,7 @@ export interface ConnectorInterface extends Component {
|
|
|
25
25
|
listNodes?(options: ListNodesOptions): Promise<ListNodesResult>;
|
|
26
26
|
loadObjectRecords?(options: LoadObjectRecordsOptions, chunks: AsyncIterable<StoreRecord[]>): Promise<LoadObjectRecordsResult>;
|
|
27
27
|
openStore?(options: OpenStoreOptions): Promise<OpenStoreResult>;
|
|
28
|
-
previewObject?(options: PreviewObjectOptions): Promise<
|
|
28
|
+
previewObject?(options: PreviewObjectOptions): Promise<PreviewObjectResult>;
|
|
29
29
|
queryRecords?(options: QueryRecordsOptions): Promise<QueryRecordsResult>;
|
|
30
30
|
removeRecords?(options: RemoveRecordsOptions): Promise<void>;
|
|
31
31
|
retrieveChunks?(options: RetrieveChunksOptions, chunk: (data: Uint8Array) => void, complete: () => void): Promise<void>;
|
|
@@ -51,12 +51,14 @@ type ConnectorItemTypeConfig = InferOutput<typeof connectorItemTypeConfigSchema>
|
|
|
51
51
|
export type ConnectorItemTypeId = InferOutput<typeof connectorItemTypeIdSchema>;
|
|
52
52
|
type ConnectorUsageConfig = InferOutput<typeof connectorUsageConfigSchema>;
|
|
53
53
|
export interface AuditObjectContentOptions extends EngineConnectorActionOptions {
|
|
54
|
+
byteLimit: number | undefined;
|
|
54
55
|
chunkSize: number | undefined;
|
|
55
56
|
dataFormatId: DataFormatId;
|
|
56
57
|
encodingId: string;
|
|
57
58
|
hasHeaders: boolean;
|
|
58
59
|
parserId: string | undefined;
|
|
59
60
|
path: string;
|
|
61
|
+
stopSignal: AbortSignal | undefined;
|
|
60
62
|
supportsTransferableStreams: boolean;
|
|
61
63
|
valueDelimiterId: ValueDelimiterId;
|
|
62
64
|
}
|
|
@@ -120,9 +122,14 @@ export interface LoadObjectRecordsResult {
|
|
|
120
122
|
}
|
|
121
123
|
export interface EngineLoadObjectRecordsOptions extends LoadObjectRecordsOptions {
|
|
122
124
|
hasHeaders: boolean;
|
|
125
|
+
headerValues: (string | null)[] | undefined;
|
|
123
126
|
sourceConnectionConfig: ConnectionConfig | LocalisedConfig<ConnectionConfig>;
|
|
124
127
|
sourceOptions: RetrieveRecordsOptions;
|
|
125
128
|
}
|
|
129
|
+
export interface EngineLoadObjectRecordsResult extends LoadObjectRecordsResult {
|
|
130
|
+
columnsHaveChanged: boolean;
|
|
131
|
+
invalidValueCounts: number[];
|
|
132
|
+
}
|
|
126
133
|
export type StoreRecord = StoreValue[];
|
|
127
134
|
export type StoreValue = bigint | boolean | Date | number | string | null;
|
|
128
135
|
export type OpenStoreOptions = EngineConnectorActionOptions;
|
|
@@ -140,6 +147,11 @@ export interface PreviewObjectOptions extends EngineConnectorActionOptions {
|
|
|
140
147
|
path: string;
|
|
141
148
|
valueDelimiterId: ValueDelimiterId | undefined;
|
|
142
149
|
}
|
|
150
|
+
export interface PreviewObjectResult {
|
|
151
|
+
contentAuditConfig: ContentAuditConfig;
|
|
152
|
+
previewConfig: PreviewConfig;
|
|
153
|
+
readConfig: ReadConfig;
|
|
154
|
+
}
|
|
143
155
|
export interface QueryRecordsOptions extends EngineConnectorActionOptions {
|
|
144
156
|
aggregates: QueryAggregate[];
|
|
145
157
|
filters: QueryFilter[];
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { FileTypeResult } from 'file-type';
|
|
2
|
-
import { AuditObjectContentOptions, ColumnAuditor, ConnectorUtilities, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
|
|
3
|
-
import { ContentAuditConfig, DataFormatId, ParsingRecord,
|
|
2
|
+
import { AuditObjectContentOptions, ColumnAuditor, ConnectorUtilities, PreviewObjectResult, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
|
|
3
|
+
import { ContentAuditConfig, DataFormatId, ParsingRecord, ValueDelimiterId } from '../../dataView';
|
|
4
4
|
import { ToolConfig } from '../tool';
|
|
5
5
|
export interface ObjectFilePreview {
|
|
6
6
|
bytes: Uint8Array;
|
|
@@ -11,7 +11,7 @@ export interface ObjectFilePreview {
|
|
|
11
11
|
isComplete: boolean;
|
|
12
12
|
text: string | undefined;
|
|
13
13
|
}
|
|
14
|
-
export declare function
|
|
14
|
+
export declare function buildPreviewObjectResult(input: {
|
|
15
15
|
asAt: number;
|
|
16
16
|
filePreview: ObjectFilePreview;
|
|
17
17
|
inferDataTypes: ConnectorUtilities['inferDataTypes'];
|
|
@@ -19,7 +19,7 @@ export declare function buildPreviewConfig(input: {
|
|
|
19
19
|
startedAt: number;
|
|
20
20
|
toolConfigs: ToolConfig[];
|
|
21
21
|
valueDelimiterId: ValueDelimiterId | undefined;
|
|
22
|
-
}): Promise<
|
|
22
|
+
}): Promise<PreviewObjectResult>;
|
|
23
23
|
export declare function retrieveRecordsWithParser(input: {
|
|
24
24
|
chunk: (typeId: RecordRetrievalTypeId, records: ParsingRecord[]) => void | Promise<void>;
|
|
25
25
|
options: RetrieveRecordsOptions;
|
|
@@ -29,7 +29,7 @@ export declare function retrieveRecordsWithParser(input: {
|
|
|
29
29
|
}): Promise<RetrieveRecordsSummary>;
|
|
30
30
|
export declare function auditObjectContentWithParser(input: {
|
|
31
31
|
constructColumnAuditor: (hasHeaders: boolean) => ColumnAuditor;
|
|
32
|
-
onProgress: (recordCount: number) => void;
|
|
32
|
+
onProgress: (recordCount: number, byteCount: number) => void;
|
|
33
33
|
options: AuditObjectContentOptions;
|
|
34
34
|
signal: AbortSignal;
|
|
35
35
|
stream: ReadableStream<Uint8Array>;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dpuse/dpuse-shared",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.171",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "Common constants, types and utilities used across DPUse projects.",
|
|
6
6
|
"license": "MIT",
|
|
@@ -64,8 +64,8 @@
|
|
|
64
64
|
"valibot": "^1.5.0"
|
|
65
65
|
},
|
|
66
66
|
"devDependencies": {
|
|
67
|
-
"@dpuse/dpuse-development": "^1.0.
|
|
68
|
-
"@dpuse/eslint-config-dpuse": "^1.0.
|
|
67
|
+
"@dpuse/dpuse-development": "^1.0.144",
|
|
68
|
+
"@dpuse/eslint-config-dpuse": "^1.0.148",
|
|
69
69
|
"@types/node": "^26.6.6",
|
|
70
70
|
"@vitest/coverage-v8": "^5.0.5",
|
|
71
71
|
"eslint": "^10.14.0",
|