@dpuse/dpuse-shared 1.0.198 → 1.0.199
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -44,66 +44,15 @@ export interface ConnectionNodeConfig extends LocalisedConfig<BaseConfig> {
|
|
|
44
44
|
typeId: 'folder' | 'object';
|
|
45
45
|
}
|
|
46
46
|
export interface ObjectColumnConfig {
|
|
47
|
-
|
|
47
|
+
label: LocaleLabel;
|
|
48
|
+
derivedFromColumnIndex?: number | undefined;
|
|
48
49
|
dataTypeId: DataTypeId;
|
|
49
50
|
dataSubtypeId: DataSubtypeId | undefined;
|
|
50
|
-
|
|
51
|
-
dateReadings?: Partial<Record<DateOrderId, DateReadingConfig>> | undefined;
|
|
51
|
+
storageTypeId: StorageTypeId | undefined;
|
|
52
52
|
decimalSeparatorId: DecimalSeparatorId | undefined;
|
|
53
|
-
|
|
54
|
-
derivedFromColumnIndex?: number | undefined;
|
|
55
|
-
distinctValueCount: number | undefined;
|
|
56
|
-
frequentValues: Record<string, number> | undefined;
|
|
57
|
-
highestValues?: [string, number][] | undefined;
|
|
58
|
-
inferenceCounts: Record<string, number>;
|
|
59
|
-
invalidValueCount: number | undefined;
|
|
60
|
-
invalidDistinctValueCount?: number | undefined;
|
|
61
|
-
invalidFrequentValues?: Record<string, number> | undefined;
|
|
62
|
-
invalidValues: Record<string, string>;
|
|
63
|
-
isFormatAmbiguous: boolean | undefined;
|
|
64
|
-
isIgnored: boolean | undefined;
|
|
65
|
-
isComplete: boolean | undefined;
|
|
66
|
-
isUnique: boolean | undefined;
|
|
67
|
-
label: LocaleLabel;
|
|
68
|
-
leadingInferenceCounts: Record<string, number>;
|
|
69
|
-
leastFrequentValues?: Record<string, number> | undefined;
|
|
70
|
-
lowestValues?: [string, number][] | undefined;
|
|
71
|
-
maxDecimals: number | undefined;
|
|
72
|
-
maxSize: number | undefined;
|
|
73
|
-
maxValue: string | undefined;
|
|
74
|
-
maxWholeDigits: number | undefined;
|
|
75
|
-
minDecimals: number | undefined;
|
|
76
|
-
minSize: number | undefined;
|
|
77
|
-
minValue: string | undefined;
|
|
78
|
-
minWholeDigits: number | undefined;
|
|
79
|
-
numericReadings?: Partial<Record<DecimalSeparatorId, NumericReadingConfig>> | undefined;
|
|
80
|
-
missingValueCount: number | undefined;
|
|
53
|
+
dateOrderId: DateOrderId | undefined;
|
|
81
54
|
nullWordsAreEmpty?: boolean | undefined;
|
|
82
|
-
nullWordValueCount?: number | undefined;
|
|
83
|
-
formats: Record<string, number>;
|
|
84
|
-
singleValueCount?: number | undefined;
|
|
85
|
-
storageTypeId: StorageTypeId | undefined;
|
|
86
|
-
validValueCount: number | undefined;
|
|
87
|
-
validValues: Record<string, string>;
|
|
88
|
-
valueBinOutliers?: ValueBinOutlierConfig | undefined;
|
|
89
|
-
valueBinScaleId?: ValueBinScaleId | undefined;
|
|
90
|
-
valueBins?: ValueBinConfig[] | undefined;
|
|
91
|
-
voidValueCount: number | undefined;
|
|
92
|
-
}
|
|
93
|
-
export interface ValueBinOutlierConfig {
|
|
94
|
-
aboveCount: number;
|
|
95
|
-
belowCount: number;
|
|
96
|
-
isEstimated: boolean;
|
|
97
|
-
}
|
|
98
|
-
export type ValueBinScaleId = 'linear' | 'log';
|
|
99
|
-
export interface ValueBinConfig {
|
|
100
|
-
count: number;
|
|
101
|
-
lowerValue: string;
|
|
102
|
-
upperValue: string;
|
|
103
55
|
}
|
|
104
|
-
export type ColumnDerivationId = 'currency' | 'unit';
|
|
105
|
-
export type NumericReadingConfig = Pick<ObjectColumnConfig, 'dataSubtypeId' | 'maxDecimals' | 'maxValue' | 'maxWholeDigits' | 'minDecimals' | 'minValue' | 'minWholeDigits' | 'storageTypeId'>;
|
|
106
|
-
export type DateReadingConfig = Pick<ObjectColumnConfig, 'formats' | 'maxValue' | 'minValue'>;
|
|
107
56
|
export type DateOrderId = 'dayFirst' | 'monthFirst';
|
|
108
57
|
export type DecimalSeparatorId = (typeof DECIMAL_SEPARATOR_IDS)[number];
|
|
109
58
|
export type StorageTypeId = 'binary' | 'boolean' | 'byte' | 'date' | 'dateTime' | 'dateTimeOffset' | 'decimal' | 'double' | 'int8' | 'int16' | 'int32' | 'int64' | 'object' | 'single' | 'string' | 'time' | 'unknown';
|
|
@@ -22,7 +22,6 @@ export interface ReadingConfig {
|
|
|
22
22
|
valueDelimiterId: ValueDelimiterId | undefined;
|
|
23
23
|
}
|
|
24
24
|
export type DataFormatId = (typeof DATA_FORMAT_CONFIGS)[number]['id'];
|
|
25
|
-
export type DataFormatSupportId = 'planned' | 'supported' | 'unsupported';
|
|
26
25
|
export type RecordDelimiterId = '\n' | '\r' | '\r\n';
|
|
27
26
|
export type ValueDelimiterId = (typeof VALUE_DELIMITER_CONFIGS)[number]['id'] | '';
|
|
28
27
|
export interface SampleConfig {
|
|
@@ -49,7 +48,7 @@ export type ParsedRecord = {
|
|
|
49
48
|
export interface ContentAuditConfig {
|
|
50
49
|
asAt: number;
|
|
51
50
|
byteCount: number;
|
|
52
|
-
columns:
|
|
51
|
+
columns: ColumnAuditConfig[];
|
|
53
52
|
commentLineCount: number | undefined;
|
|
54
53
|
emptyLineCount: number | undefined;
|
|
55
54
|
invalidFieldLengthCount: number | undefined;
|
|
@@ -59,12 +58,59 @@ export interface ContentAuditConfig {
|
|
|
59
58
|
recordCount: number;
|
|
60
59
|
sourceWaitDuration: number | undefined;
|
|
61
60
|
}
|
|
61
|
+
export interface ColumnAuditConfig extends ObjectColumnConfig {
|
|
62
|
+
inferenceCounts: Record<string, number>;
|
|
63
|
+
leadingInferenceCounts: Record<string, number>;
|
|
64
|
+
numericReadings?: Partial<Record<DecimalSeparatorId, NumericReadingConfig>> | undefined;
|
|
65
|
+
dateReadings?: Partial<Record<DateOrderId, DateReadingConfig>> | undefined;
|
|
66
|
+
isFormatAmbiguous: boolean | undefined;
|
|
67
|
+
nullWordValueCount?: number | undefined;
|
|
68
|
+
validValueCount: number | undefined;
|
|
69
|
+
invalidValueCount: number | undefined;
|
|
70
|
+
blankValueCount: number | undefined;
|
|
71
|
+
missingValueCount: number | undefined;
|
|
72
|
+
isComplete: boolean | undefined;
|
|
73
|
+
validFormats: Record<string, number>;
|
|
74
|
+
validMinSize: number | undefined;
|
|
75
|
+
validMaxSize: number | undefined;
|
|
76
|
+
validMinWholeDigits: number | undefined;
|
|
77
|
+
validMaxWholeDigits: number | undefined;
|
|
78
|
+
validMinDecimals: number | undefined;
|
|
79
|
+
validMaxDecimals: number | undefined;
|
|
80
|
+
validDistinctValueCount: number | undefined;
|
|
81
|
+
validSingleValueCount?: number | undefined;
|
|
82
|
+
invalidDistinctValueCount?: number | undefined;
|
|
83
|
+
validIsUnique: boolean | undefined;
|
|
84
|
+
validFrequentValues: Record<string, number> | undefined;
|
|
85
|
+
validLeastFrequentValues?: Record<string, number> | undefined;
|
|
86
|
+
invalidFrequentValues?: Record<string, number> | undefined;
|
|
87
|
+
invalidValues: Record<string, string>;
|
|
88
|
+
validLowestValues?: [string, number][] | undefined;
|
|
89
|
+
validHighestValues?: [string, number][] | undefined;
|
|
90
|
+
validValueBins?: ValueBinConfig[] | undefined;
|
|
91
|
+
validValueBinScaleId?: ValueBinScaleId | undefined;
|
|
92
|
+
validValueBinOutliers?: ValueBinOutlierConfig | undefined;
|
|
93
|
+
}
|
|
94
|
+
export type NumericReadingConfig = Pick<ColumnAuditConfig, 'dataSubtypeId' | 'validMaxDecimals' | 'validMaxWholeDigits' | 'validMinDecimals' | 'validMinWholeDigits' | 'storageTypeId'>;
|
|
95
|
+
export type DateReadingConfig = Pick<ColumnAuditConfig, 'validFormats'>;
|
|
96
|
+
export interface ValueBinConfig {
|
|
97
|
+
count: number;
|
|
98
|
+
lowerValue: string;
|
|
99
|
+
upperValue: string;
|
|
100
|
+
}
|
|
101
|
+
export type ValueBinScaleId = 'linear' | 'log';
|
|
102
|
+
export interface ValueBinOutlierConfig {
|
|
103
|
+
aboveCount: number;
|
|
104
|
+
belowCount: number;
|
|
105
|
+
isEstimated: boolean;
|
|
106
|
+
}
|
|
62
107
|
export type ColumnChoiceConfigs = Partial<Record<string, ColumnChoiceConfig>>;
|
|
63
108
|
export interface ColumnChoiceConfig {
|
|
64
109
|
dataTypeId: DataTypeId | undefined;
|
|
65
110
|
dataSubtypeId: DataSubtypeId | undefined;
|
|
66
111
|
dateOrderId: DateOrderId | undefined;
|
|
67
112
|
decimalSeparatorId: DecimalSeparatorId | undefined;
|
|
113
|
+
isIgnored: boolean | undefined;
|
|
68
114
|
nullWordsAreEmpty: boolean | undefined;
|
|
69
115
|
}
|
|
70
116
|
type DataTypeConfig = (typeof DATA_TYPE_CONFIGS)[number];
|
|
@@ -3,15 +3,13 @@ import { Component } from '../..';
|
|
|
3
3
|
import { EngineConnectorActionOptions } from '../engine';
|
|
4
4
|
import { LocalisedConfig } from '../../../locale';
|
|
5
5
|
import { ToolConfig } from '../tool';
|
|
6
|
+
import { ColumnAuditConfig, ContentAuditConfig, DataFormatId, ParsedRecord, ReadingConfig, SampleConfig, ValueDelimiterId } from '../../dataView';
|
|
6
7
|
import { ConnectionConfig, ConnectionDescriptionConfig, ConnectionNodeConfig, ObjectColumnConfig } from '../../connection';
|
|
7
8
|
import { connectorActionNameSchema, connectorCategoryConfigSchema, connectorConfigSchema, connectorItemTypeConfigSchema, connectorItemTypeIdSchema, connectorUsageConfigSchema, connectorUsageIdSchema } from './connectorConfig.schema';
|
|
8
|
-
import { ContentAuditConfig, DataFormatId, ParsedRecord, ReadingConfig, SampleConfig, ValueDelimiterId } from '../../dataView';
|
|
9
9
|
export { connectorConfigSchema } from './connectorConfig.schema';
|
|
10
10
|
export { auditObjectContentWithParser, buildPreviewObjectResult, type ObjectFilePreview, retrieveRecordsWithParser } from './objectParsing';
|
|
11
11
|
export interface ConnectorInterface extends Component {
|
|
12
|
-
abortController: AbortController | undefined;
|
|
13
12
|
readonly config: ConnectorConfig;
|
|
14
|
-
readonly toolConfigs: ToolConfig[];
|
|
15
13
|
abortOperation?(): void;
|
|
16
14
|
auditObjectContent?(options: AuditObjectContentOptions, chunk: (recordCount: number, byteCount: number) => void): Promise<AuditObjectContentResult>;
|
|
17
15
|
closeStore?(options: CloseStoreOptions): Promise<void>;
|
|
@@ -45,7 +43,7 @@ export interface ColumnAuditor {
|
|
|
45
43
|
completeAudit: () => InferenceSummary;
|
|
46
44
|
}
|
|
47
45
|
export interface InferenceSummary {
|
|
48
|
-
columnConfigs:
|
|
46
|
+
columnConfigs: ColumnAuditConfig[];
|
|
49
47
|
hasHeaders: boolean;
|
|
50
48
|
}
|
|
51
49
|
export type ConnectorConfig = InferOutput<typeof connectorConfigSchema>;
|
|
@@ -126,6 +124,7 @@ export interface LoadObjectRecordsResult {
|
|
|
126
124
|
}
|
|
127
125
|
export interface EngineLoadObjectRecordsOptions extends LoadObjectRecordsOptions {
|
|
128
126
|
hasHeaders: boolean;
|
|
127
|
+
ignoredColumnIndexes: number[];
|
|
129
128
|
headerValues: (string | null)[] | undefined;
|
|
130
129
|
sourceConnectionConfig: ConnectionConfig | LocalisedConfig<ConnectionConfig>;
|
|
131
130
|
sourceOptions: RetrieveRecordsOptions;
|