@dpuse/dpuse-shared 1.0.198 → 1.0.200
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/dpuse-shared.es.js +7 -5
- package/dist/dpuse-shared.es.js.map +1 -1
- package/dist/types/src/component/connection/index.d.ts +1 -59
- package/dist/types/src/component/dataView/index.d.ts +58 -3
- package/dist/types/src/component/module/connector/index.d.ts +6 -8
- package/package.json +1 -1
|
@@ -44,68 +44,10 @@ export interface ConnectionNodeConfig extends LocalisedConfig<BaseConfig> {
|
|
|
44
44
|
typeId: 'folder' | 'object';
|
|
45
45
|
}
|
|
46
46
|
export interface ObjectColumnConfig {
|
|
47
|
-
|
|
47
|
+
label: LocaleLabel;
|
|
48
48
|
dataTypeId: DataTypeId;
|
|
49
49
|
dataSubtypeId: DataSubtypeId | undefined;
|
|
50
|
-
dateOrderId: DateOrderId | undefined;
|
|
51
|
-
dateReadings?: Partial<Record<DateOrderId, DateReadingConfig>> | undefined;
|
|
52
|
-
decimalSeparatorId: DecimalSeparatorId | undefined;
|
|
53
|
-
derivationId?: ColumnDerivationId | undefined;
|
|
54
|
-
derivedFromColumnIndex?: number | undefined;
|
|
55
|
-
distinctValueCount: number | undefined;
|
|
56
|
-
frequentValues: Record<string, number> | undefined;
|
|
57
|
-
highestValues?: [string, number][] | undefined;
|
|
58
|
-
inferenceCounts: Record<string, number>;
|
|
59
|
-
invalidValueCount: number | undefined;
|
|
60
|
-
invalidDistinctValueCount?: number | undefined;
|
|
61
|
-
invalidFrequentValues?: Record<string, number> | undefined;
|
|
62
|
-
invalidValues: Record<string, string>;
|
|
63
|
-
isFormatAmbiguous: boolean | undefined;
|
|
64
|
-
isIgnored: boolean | undefined;
|
|
65
|
-
isComplete: boolean | undefined;
|
|
66
|
-
isUnique: boolean | undefined;
|
|
67
|
-
label: LocaleLabel;
|
|
68
|
-
leadingInferenceCounts: Record<string, number>;
|
|
69
|
-
leastFrequentValues?: Record<string, number> | undefined;
|
|
70
|
-
lowestValues?: [string, number][] | undefined;
|
|
71
|
-
maxDecimals: number | undefined;
|
|
72
|
-
maxSize: number | undefined;
|
|
73
|
-
maxValue: string | undefined;
|
|
74
|
-
maxWholeDigits: number | undefined;
|
|
75
|
-
minDecimals: number | undefined;
|
|
76
|
-
minSize: number | undefined;
|
|
77
|
-
minValue: string | undefined;
|
|
78
|
-
minWholeDigits: number | undefined;
|
|
79
|
-
numericReadings?: Partial<Record<DecimalSeparatorId, NumericReadingConfig>> | undefined;
|
|
80
|
-
missingValueCount: number | undefined;
|
|
81
|
-
nullWordsAreEmpty?: boolean | undefined;
|
|
82
|
-
nullWordValueCount?: number | undefined;
|
|
83
|
-
formats: Record<string, number>;
|
|
84
|
-
singleValueCount?: number | undefined;
|
|
85
50
|
storageTypeId: StorageTypeId | undefined;
|
|
86
|
-
validValueCount: number | undefined;
|
|
87
|
-
validValues: Record<string, string>;
|
|
88
|
-
valueBinOutliers?: ValueBinOutlierConfig | undefined;
|
|
89
|
-
valueBinScaleId?: ValueBinScaleId | undefined;
|
|
90
|
-
valueBins?: ValueBinConfig[] | undefined;
|
|
91
|
-
voidValueCount: number | undefined;
|
|
92
|
-
}
|
|
93
|
-
export interface ValueBinOutlierConfig {
|
|
94
|
-
aboveCount: number;
|
|
95
|
-
belowCount: number;
|
|
96
|
-
isEstimated: boolean;
|
|
97
|
-
}
|
|
98
|
-
export type ValueBinScaleId = 'linear' | 'log';
|
|
99
|
-
export interface ValueBinConfig {
|
|
100
|
-
count: number;
|
|
101
|
-
lowerValue: string;
|
|
102
|
-
upperValue: string;
|
|
103
51
|
}
|
|
104
|
-
export type ColumnDerivationId = 'currency' | 'unit';
|
|
105
|
-
export type NumericReadingConfig = Pick<ObjectColumnConfig, 'dataSubtypeId' | 'maxDecimals' | 'maxValue' | 'maxWholeDigits' | 'minDecimals' | 'minValue' | 'minWholeDigits' | 'storageTypeId'>;
|
|
106
|
-
export type DateReadingConfig = Pick<ObjectColumnConfig, 'formats' | 'maxValue' | 'minValue'>;
|
|
107
|
-
export type DateOrderId = 'dayFirst' | 'monthFirst';
|
|
108
|
-
export type DecimalSeparatorId = (typeof DECIMAL_SEPARATOR_IDS)[number];
|
|
109
52
|
export type StorageTypeId = 'binary' | 'boolean' | 'byte' | 'date' | 'dateTime' | 'dateTimeOffset' | 'decimal' | 'double' | 'int8' | 'int16' | 'int32' | 'int64' | 'object' | 'single' | 'string' | 'time' | 'unknown';
|
|
110
|
-
export declare const DECIMAL_SEPARATOR_IDS: readonly [",", "."];
|
|
111
53
|
export {};
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { FileTypeResult } from 'file-type';
|
|
2
2
|
import { ComponentInstanceConfig } from '..';
|
|
3
|
-
import { ConnectionNodeConfig,
|
|
3
|
+
import { ConnectionNodeConfig, ObjectColumnConfig } from '../connection';
|
|
4
4
|
export interface DataViewConfig extends ComponentInstanceConfig {
|
|
5
5
|
typeId: 'dataView';
|
|
6
6
|
connectionId: string;
|
|
@@ -22,7 +22,6 @@ export interface ReadingConfig {
|
|
|
22
22
|
valueDelimiterId: ValueDelimiterId | undefined;
|
|
23
23
|
}
|
|
24
24
|
export type DataFormatId = (typeof DATA_FORMAT_CONFIGS)[number]['id'];
|
|
25
|
-
export type DataFormatSupportId = 'planned' | 'supported' | 'unsupported';
|
|
26
25
|
export type RecordDelimiterId = '\n' | '\r' | '\r\n';
|
|
27
26
|
export type ValueDelimiterId = (typeof VALUE_DELIMITER_CONFIGS)[number]['id'] | '';
|
|
28
27
|
export interface SampleConfig {
|
|
@@ -49,7 +48,7 @@ export type ParsedRecord = {
|
|
|
49
48
|
export interface ContentAuditConfig {
|
|
50
49
|
asAt: number;
|
|
51
50
|
byteCount: number;
|
|
52
|
-
columns:
|
|
51
|
+
columns: ColumnAuditConfig[];
|
|
53
52
|
commentLineCount: number | undefined;
|
|
54
53
|
emptyLineCount: number | undefined;
|
|
55
54
|
invalidFieldLengthCount: number | undefined;
|
|
@@ -59,12 +58,67 @@ export interface ContentAuditConfig {
|
|
|
59
58
|
recordCount: number;
|
|
60
59
|
sourceWaitDuration: number | undefined;
|
|
61
60
|
}
|
|
61
|
+
export interface ColumnReadingConfig extends ObjectColumnConfig {
|
|
62
|
+
derivedFromColumnIndex?: number | undefined;
|
|
63
|
+
decimalSeparatorId: DecimalSeparatorId | undefined;
|
|
64
|
+
dateOrderId: DateOrderId | undefined;
|
|
65
|
+
nullWordsAreEmpty?: boolean | undefined;
|
|
66
|
+
}
|
|
67
|
+
export type DateOrderId = 'dayFirst' | 'monthFirst';
|
|
68
|
+
export type DecimalSeparatorId = (typeof DECIMAL_SEPARATOR_IDS)[number];
|
|
69
|
+
export interface ColumnAuditConfig extends ColumnReadingConfig {
|
|
70
|
+
inferenceCounts: Record<string, number>;
|
|
71
|
+
leadingInferenceCounts: Record<string, number>;
|
|
72
|
+
numericReadings?: Partial<Record<DecimalSeparatorId, NumericReadingConfig>> | undefined;
|
|
73
|
+
dateReadings?: Partial<Record<DateOrderId, DateReadingConfig>> | undefined;
|
|
74
|
+
isFormatAmbiguous: boolean | undefined;
|
|
75
|
+
nullWordValueCount?: number | undefined;
|
|
76
|
+
validValueCount: number | undefined;
|
|
77
|
+
invalidValueCount: number | undefined;
|
|
78
|
+
blankValueCount: number | undefined;
|
|
79
|
+
missingValueCount: number | undefined;
|
|
80
|
+
isComplete: boolean | undefined;
|
|
81
|
+
validFormats: Record<string, number>;
|
|
82
|
+
validMinSize: number | undefined;
|
|
83
|
+
validMaxSize: number | undefined;
|
|
84
|
+
validMinWholeDigits: number | undefined;
|
|
85
|
+
validMaxWholeDigits: number | undefined;
|
|
86
|
+
validMinDecimals: number | undefined;
|
|
87
|
+
validMaxDecimals: number | undefined;
|
|
88
|
+
validDistinctValueCount: number | undefined;
|
|
89
|
+
validSingleValueCount?: number | undefined;
|
|
90
|
+
invalidDistinctValueCount?: number | undefined;
|
|
91
|
+
validIsUnique: boolean | undefined;
|
|
92
|
+
validFrequentValues: Record<string, number> | undefined;
|
|
93
|
+
validLeastFrequentValues?: Record<string, number> | undefined;
|
|
94
|
+
invalidFrequentValues?: Record<string, number> | undefined;
|
|
95
|
+
invalidValues: Record<string, string>;
|
|
96
|
+
validLowestValues?: [string, number][] | undefined;
|
|
97
|
+
validHighestValues?: [string, number][] | undefined;
|
|
98
|
+
validValueBins?: ValueBinConfig[] | undefined;
|
|
99
|
+
validValueBinScaleId?: ValueBinScaleId | undefined;
|
|
100
|
+
validValueBinOutliers?: ValueBinOutlierConfig | undefined;
|
|
101
|
+
}
|
|
102
|
+
export type NumericReadingConfig = Pick<ColumnAuditConfig, 'dataSubtypeId' | 'validMaxDecimals' | 'validMaxWholeDigits' | 'validMinDecimals' | 'validMinWholeDigits' | 'storageTypeId'>;
|
|
103
|
+
export type DateReadingConfig = Pick<ColumnAuditConfig, 'validFormats'>;
|
|
104
|
+
export interface ValueBinConfig {
|
|
105
|
+
count: number;
|
|
106
|
+
lowerValue: string;
|
|
107
|
+
upperValue: string;
|
|
108
|
+
}
|
|
109
|
+
export type ValueBinScaleId = 'linear' | 'log';
|
|
110
|
+
export interface ValueBinOutlierConfig {
|
|
111
|
+
aboveCount: number;
|
|
112
|
+
belowCount: number;
|
|
113
|
+
isEstimated: boolean;
|
|
114
|
+
}
|
|
62
115
|
export type ColumnChoiceConfigs = Partial<Record<string, ColumnChoiceConfig>>;
|
|
63
116
|
export interface ColumnChoiceConfig {
|
|
64
117
|
dataTypeId: DataTypeId | undefined;
|
|
65
118
|
dataSubtypeId: DataSubtypeId | undefined;
|
|
66
119
|
dateOrderId: DateOrderId | undefined;
|
|
67
120
|
decimalSeparatorId: DecimalSeparatorId | undefined;
|
|
121
|
+
isIgnored: boolean | undefined;
|
|
68
122
|
nullWordsAreEmpty: boolean | undefined;
|
|
69
123
|
}
|
|
70
124
|
type DataTypeConfig = (typeof DATA_TYPE_CONFIGS)[number];
|
|
@@ -202,5 +256,6 @@ export declare const DATA_TYPE_CONFIGS: readonly [{
|
|
|
202
256
|
readonly isChoosable: false;
|
|
203
257
|
readonly subtypeIds: readonly [];
|
|
204
258
|
}];
|
|
259
|
+
export declare const DECIMAL_SEPARATOR_IDS: readonly [",", "."];
|
|
205
260
|
export declare function getValueDelimiterCharacter(valueDelimiterId: ValueDelimiterId): string;
|
|
206
261
|
export {};
|
|
@@ -1,17 +1,13 @@
|
|
|
1
1
|
import { InferOutput } from 'valibot';
|
|
2
|
-
import { Component } from '../..';
|
|
3
2
|
import { EngineConnectorActionOptions } from '../engine';
|
|
4
3
|
import { LocalisedConfig } from '../../../locale';
|
|
5
4
|
import { ToolConfig } from '../tool';
|
|
5
|
+
import { ColumnAuditConfig, ColumnReadingConfig, ContentAuditConfig, DataFormatId, ParsedRecord, ReadingConfig, SampleConfig, ValueDelimiterId } from '../../dataView';
|
|
6
6
|
import { ConnectionConfig, ConnectionDescriptionConfig, ConnectionNodeConfig, ObjectColumnConfig } from '../../connection';
|
|
7
7
|
import { connectorActionNameSchema, connectorCategoryConfigSchema, connectorConfigSchema, connectorItemTypeConfigSchema, connectorItemTypeIdSchema, connectorUsageConfigSchema, connectorUsageIdSchema } from './connectorConfig.schema';
|
|
8
|
-
import { ContentAuditConfig, DataFormatId, ParsedRecord, ReadingConfig, SampleConfig, ValueDelimiterId } from '../../dataView';
|
|
9
8
|
export { connectorConfigSchema } from './connectorConfig.schema';
|
|
10
9
|
export { auditObjectContentWithParser, buildPreviewObjectResult, type ObjectFilePreview, retrieveRecordsWithParser } from './objectParsing';
|
|
11
|
-
export interface ConnectorInterface
|
|
12
|
-
abortController: AbortController | undefined;
|
|
13
|
-
readonly config: ConnectorConfig;
|
|
14
|
-
readonly toolConfigs: ToolConfig[];
|
|
10
|
+
export interface ConnectorInterface {
|
|
15
11
|
abortOperation?(): void;
|
|
16
12
|
auditObjectContent?(options: AuditObjectContentOptions, chunk: (recordCount: number, byteCount: number) => void): Promise<AuditObjectContentResult>;
|
|
17
13
|
closeStore?(options: CloseStoreOptions): Promise<void>;
|
|
@@ -45,7 +41,7 @@ export interface ColumnAuditor {
|
|
|
45
41
|
completeAudit: () => InferenceSummary;
|
|
46
42
|
}
|
|
47
43
|
export interface InferenceSummary {
|
|
48
|
-
columnConfigs:
|
|
44
|
+
columnConfigs: ColumnAuditConfig[];
|
|
49
45
|
hasHeaders: boolean;
|
|
50
46
|
}
|
|
51
47
|
export type ConnectorConfig = InferOutput<typeof connectorConfigSchema>;
|
|
@@ -125,7 +121,9 @@ export interface LoadObjectRecordsResult {
|
|
|
125
121
|
recordCount: number;
|
|
126
122
|
}
|
|
127
123
|
export interface EngineLoadObjectRecordsOptions extends LoadObjectRecordsOptions {
|
|
124
|
+
columnConfigs: ColumnReadingConfig[];
|
|
128
125
|
hasHeaders: boolean;
|
|
126
|
+
ignoredColumnIndexes: number[];
|
|
129
127
|
headerValues: (string | null)[] | undefined;
|
|
130
128
|
sourceConnectionConfig: ConnectionConfig | LocalisedConfig<ConnectionConfig>;
|
|
131
129
|
sourceOptions: RetrieveRecordsOptions;
|
|
@@ -189,7 +187,7 @@ export interface RecountPreviewColumnsOptions extends EngineConnectorActionOptio
|
|
|
189
187
|
parsedRecords: ParsedRecord[];
|
|
190
188
|
}
|
|
191
189
|
export interface ClassifyPreviewValuesOptions extends EngineConnectorActionOptions {
|
|
192
|
-
columnConfigs:
|
|
190
|
+
columnConfigs: ColumnReadingConfig[];
|
|
193
191
|
parsedRecords: ParsedRecord[];
|
|
194
192
|
}
|
|
195
193
|
export type PreviewValueStateId = 'blank' | 'invalid' | 'missing' | 'valid';
|