@dpuse/dpuse-shared 1.0.170 → 1.0.171
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -9
- package/dist/dpuse-shared.es.js +783 -705
- package/dist/dpuse-shared.es.js.map +1 -1
- package/dist/types/src/component/dataView/index.d.ts +14 -15
- package/dist/types/src/component/module/connector/index.d.ts +11 -4
- package/dist/types/src/component/module/connector/objectParsing.d.ts +5 -5
- package/package.json +3 -3
|
@@ -5,51 +5,48 @@ export interface DataViewConfig extends ComponentInstanceConfig {
|
|
|
5
5
|
typeId: 'dataView';
|
|
6
6
|
connectionId: string;
|
|
7
7
|
connectionNodeConfig: ConnectionNodeConfig | undefined;
|
|
8
|
-
|
|
8
|
+
readConfig: ReadConfig | undefined;
|
|
9
|
+
previewConfig: PreviewConfig | undefined;
|
|
10
|
+
contentAuditConfig: StoredContentAuditConfig | undefined;
|
|
9
11
|
columnSetConfig: ColumnSetConfig | undefined;
|
|
10
|
-
auditSummaryConfig: AuditSummaryConfig | undefined;
|
|
11
12
|
relationshipsAuditConfig: RelationshipsAuditConfig | undefined;
|
|
12
13
|
}
|
|
13
14
|
export interface ColumnSetConfig {
|
|
14
15
|
asAt: number;
|
|
15
16
|
columnConfigs: DataViewColumnConfig[];
|
|
16
|
-
sourceId: ColumnSetSourceId;
|
|
17
17
|
}
|
|
18
|
-
export type ColumnSetSourceId = 'audit' | 'preview';
|
|
19
18
|
export interface DataViewColumnConfig extends ObjectColumnConfig {
|
|
20
19
|
chosenDataTypeId?: DataTypeId | undefined;
|
|
21
20
|
}
|
|
22
|
-
export interface
|
|
23
|
-
asAt: number;
|
|
21
|
+
export interface ReadConfig {
|
|
24
22
|
commentMarkId?: string | undefined;
|
|
25
23
|
commentMarkOtherCharSeq?: string | undefined;
|
|
26
|
-
columnConfigs: ObjectColumnConfig[];
|
|
27
24
|
dataFormatId: DataFormatId;
|
|
28
|
-
duration: number;
|
|
29
25
|
encodingConfidenceLevel: number | undefined;
|
|
30
26
|
encodingId: string | undefined;
|
|
31
|
-
errorMessage?: string | undefined;
|
|
32
27
|
fileType: FileTypeResult | undefined;
|
|
33
28
|
hasHeaders: boolean | undefined;
|
|
34
|
-
issues?: PreviewIssue[];
|
|
35
29
|
linesToSkipAtStart?: number | undefined;
|
|
36
|
-
parsedRecords: ParsingRecord[];
|
|
37
30
|
parserId?: string | undefined;
|
|
38
31
|
quoteEscapeChar?: string | undefined;
|
|
39
32
|
quoteMarkId?: string | undefined;
|
|
40
33
|
quoteMarkOtherCharSeq?: string | undefined;
|
|
41
34
|
recordDelimiterId: RecordDelimiterId | undefined;
|
|
42
35
|
recordDelimiterOtherCharSeq?: string | undefined;
|
|
43
|
-
size: number | undefined;
|
|
44
36
|
skipEmptyLines?: boolean | undefined;
|
|
45
37
|
skipLinesWithEmptyValues?: boolean | undefined;
|
|
46
38
|
skipLinesWithErrors?: boolean | undefined;
|
|
47
|
-
text: string | undefined;
|
|
48
39
|
valueDelimiterId: ValueDelimiterId | undefined;
|
|
49
40
|
valueDelimiterOtherCharSeq?: string | undefined;
|
|
50
41
|
valueTrimMethodId?: ValueTrimMethodId | undefined;
|
|
51
42
|
}
|
|
52
|
-
export
|
|
43
|
+
export interface PreviewConfig {
|
|
44
|
+
errorMessage?: string | undefined;
|
|
45
|
+
issues?: PreviewIssue[];
|
|
46
|
+
parsedRecords: ParsingRecord[];
|
|
47
|
+
size: number | undefined;
|
|
48
|
+
text: string | undefined;
|
|
49
|
+
}
|
|
53
50
|
export interface PreviewIssue {
|
|
54
51
|
endLineNumber?: number | undefined;
|
|
55
52
|
fieldNumber?: number | undefined;
|
|
@@ -72,16 +69,18 @@ export type ValueDelimiterId = '' | ':' | ',' | '!' | '0x1E' | ';' | ' ' | '\t'
|
|
|
72
69
|
type ValueTrimMethodId = 'both' | 'left' | 'right' | 'none';
|
|
73
70
|
export interface ContentAuditConfig {
|
|
74
71
|
asAt: number;
|
|
72
|
+
byteCount: number;
|
|
75
73
|
columns: ObjectColumnConfig[];
|
|
76
74
|
commentLineCount: number | undefined;
|
|
77
75
|
emptyLineCount: number | undefined;
|
|
78
76
|
invalidFieldLengthCount: number | undefined;
|
|
79
77
|
duration: number;
|
|
78
|
+
isComplete: boolean;
|
|
80
79
|
lineCount: number | undefined;
|
|
81
80
|
recordCount: number;
|
|
82
81
|
sourceWaitDuration: number | undefined;
|
|
83
82
|
}
|
|
84
|
-
export type
|
|
83
|
+
export type StoredContentAuditConfig = Omit<ContentAuditConfig, 'columns'>;
|
|
85
84
|
interface RelationshipsAuditConfig {
|
|
86
85
|
placeholder: string;
|
|
87
86
|
}
|
|
@@ -5,15 +5,15 @@ import { LocalisedConfig } from '../../../locale';
|
|
|
5
5
|
import { ToolConfig } from '../tool';
|
|
6
6
|
import { ConnectionConfig, ConnectionDescriptionConfig, ConnectionNodeConfig, ObjectColumnConfig } from '../../connection';
|
|
7
7
|
import { connectorActionNameSchema, connectorCategoryConfigSchema, connectorConfigSchema, connectorItemTypeConfigSchema, connectorItemTypeIdSchema, connectorUsageConfigSchema, connectorUsageIdSchema } from './connectorConfig.schema';
|
|
8
|
-
import { ContentAuditConfig, DataFormatId, InferenceRecord, InferenceSummary, ParsingRecord, PreviewConfig, ValueDelimiterId } from '../../dataView';
|
|
8
|
+
import { ContentAuditConfig, DataFormatId, InferenceRecord, InferenceSummary, ParsingRecord, PreviewConfig, ReadConfig, ValueDelimiterId } from '../../dataView';
|
|
9
9
|
export { connectorConfigSchema } from './connectorConfig.schema';
|
|
10
|
-
export { auditObjectContentWithParser,
|
|
10
|
+
export { auditObjectContentWithParser, buildPreviewObjectResult, type ObjectFilePreview, retrieveRecordsWithParser } from './objectParsing';
|
|
11
11
|
export interface ConnectorInterface extends Component {
|
|
12
12
|
abortController: AbortController | undefined;
|
|
13
13
|
readonly config: ConnectorConfig;
|
|
14
14
|
readonly toolConfigs: ToolConfig[];
|
|
15
15
|
abortOperation?(): void;
|
|
16
|
-
auditObjectContent?(options: AuditObjectContentOptions, chunk: (
|
|
16
|
+
auditObjectContent?(options: AuditObjectContentOptions, chunk: (recordCount: number, byteCount: number) => void): Promise<AuditObjectContentResult>;
|
|
17
17
|
closeStore?(options: CloseStoreOptions): Promise<void>;
|
|
18
18
|
createObject?(options: CreateObjectOptions): Promise<void>;
|
|
19
19
|
describeConnection?(options: DescribeConnectionOptions): Promise<DescribeConnectionResult>;
|
|
@@ -25,7 +25,7 @@ export interface ConnectorInterface extends Component {
|
|
|
25
25
|
listNodes?(options: ListNodesOptions): Promise<ListNodesResult>;
|
|
26
26
|
loadObjectRecords?(options: LoadObjectRecordsOptions, chunks: AsyncIterable<StoreRecord[]>): Promise<LoadObjectRecordsResult>;
|
|
27
27
|
openStore?(options: OpenStoreOptions): Promise<OpenStoreResult>;
|
|
28
|
-
previewObject?(options: PreviewObjectOptions): Promise<
|
|
28
|
+
previewObject?(options: PreviewObjectOptions): Promise<PreviewObjectResult>;
|
|
29
29
|
queryRecords?(options: QueryRecordsOptions): Promise<QueryRecordsResult>;
|
|
30
30
|
removeRecords?(options: RemoveRecordsOptions): Promise<void>;
|
|
31
31
|
retrieveChunks?(options: RetrieveChunksOptions, chunk: (data: Uint8Array) => void, complete: () => void): Promise<void>;
|
|
@@ -51,12 +51,14 @@ type ConnectorItemTypeConfig = InferOutput<typeof connectorItemTypeConfigSchema>
|
|
|
51
51
|
export type ConnectorItemTypeId = InferOutput<typeof connectorItemTypeIdSchema>;
|
|
52
52
|
type ConnectorUsageConfig = InferOutput<typeof connectorUsageConfigSchema>;
|
|
53
53
|
export interface AuditObjectContentOptions extends EngineConnectorActionOptions {
|
|
54
|
+
byteLimit: number | undefined;
|
|
54
55
|
chunkSize: number | undefined;
|
|
55
56
|
dataFormatId: DataFormatId;
|
|
56
57
|
encodingId: string;
|
|
57
58
|
hasHeaders: boolean;
|
|
58
59
|
parserId: string | undefined;
|
|
59
60
|
path: string;
|
|
61
|
+
stopSignal: AbortSignal | undefined;
|
|
60
62
|
supportsTransferableStreams: boolean;
|
|
61
63
|
valueDelimiterId: ValueDelimiterId;
|
|
62
64
|
}
|
|
@@ -145,6 +147,11 @@ export interface PreviewObjectOptions extends EngineConnectorActionOptions {
|
|
|
145
147
|
path: string;
|
|
146
148
|
valueDelimiterId: ValueDelimiterId | undefined;
|
|
147
149
|
}
|
|
150
|
+
export interface PreviewObjectResult {
|
|
151
|
+
contentAuditConfig: ContentAuditConfig;
|
|
152
|
+
previewConfig: PreviewConfig;
|
|
153
|
+
readConfig: ReadConfig;
|
|
154
|
+
}
|
|
148
155
|
export interface QueryRecordsOptions extends EngineConnectorActionOptions {
|
|
149
156
|
aggregates: QueryAggregate[];
|
|
150
157
|
filters: QueryFilter[];
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { FileTypeResult } from 'file-type';
|
|
2
|
-
import { AuditObjectContentOptions, ColumnAuditor, ConnectorUtilities, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
|
|
3
|
-
import { ContentAuditConfig, DataFormatId, ParsingRecord,
|
|
2
|
+
import { AuditObjectContentOptions, ColumnAuditor, ConnectorUtilities, PreviewObjectResult, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
|
|
3
|
+
import { ContentAuditConfig, DataFormatId, ParsingRecord, ValueDelimiterId } from '../../dataView';
|
|
4
4
|
import { ToolConfig } from '../tool';
|
|
5
5
|
export interface ObjectFilePreview {
|
|
6
6
|
bytes: Uint8Array;
|
|
@@ -11,7 +11,7 @@ export interface ObjectFilePreview {
|
|
|
11
11
|
isComplete: boolean;
|
|
12
12
|
text: string | undefined;
|
|
13
13
|
}
|
|
14
|
-
export declare function
|
|
14
|
+
export declare function buildPreviewObjectResult(input: {
|
|
15
15
|
asAt: number;
|
|
16
16
|
filePreview: ObjectFilePreview;
|
|
17
17
|
inferDataTypes: ConnectorUtilities['inferDataTypes'];
|
|
@@ -19,7 +19,7 @@ export declare function buildPreviewConfig(input: {
|
|
|
19
19
|
startedAt: number;
|
|
20
20
|
toolConfigs: ToolConfig[];
|
|
21
21
|
valueDelimiterId: ValueDelimiterId | undefined;
|
|
22
|
-
}): Promise<
|
|
22
|
+
}): Promise<PreviewObjectResult>;
|
|
23
23
|
export declare function retrieveRecordsWithParser(input: {
|
|
24
24
|
chunk: (typeId: RecordRetrievalTypeId, records: ParsingRecord[]) => void | Promise<void>;
|
|
25
25
|
options: RetrieveRecordsOptions;
|
|
@@ -29,7 +29,7 @@ export declare function retrieveRecordsWithParser(input: {
|
|
|
29
29
|
}): Promise<RetrieveRecordsSummary>;
|
|
30
30
|
export declare function auditObjectContentWithParser(input: {
|
|
31
31
|
constructColumnAuditor: (hasHeaders: boolean) => ColumnAuditor;
|
|
32
|
-
onProgress: (recordCount: number) => void;
|
|
32
|
+
onProgress: (recordCount: number, byteCount: number) => void;
|
|
33
33
|
options: AuditObjectContentOptions;
|
|
34
34
|
signal: AbortSignal;
|
|
35
35
|
stream: ReadableStream<Uint8Array>;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dpuse/dpuse-shared",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.171",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "Common constants, types and utilities used across DPUse projects.",
|
|
6
6
|
"license": "MIT",
|
|
@@ -64,8 +64,8 @@
|
|
|
64
64
|
"valibot": "^1.5.0"
|
|
65
65
|
},
|
|
66
66
|
"devDependencies": {
|
|
67
|
-
"@dpuse/dpuse-development": "^1.0.
|
|
68
|
-
"@dpuse/eslint-config-dpuse": "^1.0.
|
|
67
|
+
"@dpuse/dpuse-development": "^1.0.144",
|
|
68
|
+
"@dpuse/eslint-config-dpuse": "^1.0.148",
|
|
69
69
|
"@types/node": "^26.6.6",
|
|
70
70
|
"@vitest/coverage-v8": "^5.0.5",
|
|
71
71
|
"eslint": "^10.14.0",
|