@dpuse/dpuse-shared 1.0.198 → 1.0.199

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -44,66 +44,15 @@ export interface ConnectionNodeConfig extends LocalisedConfig<BaseConfig> {
44
44
  typeId: 'folder' | 'object';
45
45
  }
46
46
  export interface ObjectColumnConfig {
47
- blankValueCount: number | undefined;
47
+ label: LocaleLabel;
48
+ derivedFromColumnIndex?: number | undefined;
48
49
  dataTypeId: DataTypeId;
49
50
  dataSubtypeId: DataSubtypeId | undefined;
50
- dateOrderId: DateOrderId | undefined;
51
- dateReadings?: Partial<Record<DateOrderId, DateReadingConfig>> | undefined;
51
+ storageTypeId: StorageTypeId | undefined;
52
52
  decimalSeparatorId: DecimalSeparatorId | undefined;
53
- derivationId?: ColumnDerivationId | undefined;
54
- derivedFromColumnIndex?: number | undefined;
55
- distinctValueCount: number | undefined;
56
- frequentValues: Record<string, number> | undefined;
57
- highestValues?: [string, number][] | undefined;
58
- inferenceCounts: Record<string, number>;
59
- invalidValueCount: number | undefined;
60
- invalidDistinctValueCount?: number | undefined;
61
- invalidFrequentValues?: Record<string, number> | undefined;
62
- invalidValues: Record<string, string>;
63
- isFormatAmbiguous: boolean | undefined;
64
- isIgnored: boolean | undefined;
65
- isComplete: boolean | undefined;
66
- isUnique: boolean | undefined;
67
- label: LocaleLabel;
68
- leadingInferenceCounts: Record<string, number>;
69
- leastFrequentValues?: Record<string, number> | undefined;
70
- lowestValues?: [string, number][] | undefined;
71
- maxDecimals: number | undefined;
72
- maxSize: number | undefined;
73
- maxValue: string | undefined;
74
- maxWholeDigits: number | undefined;
75
- minDecimals: number | undefined;
76
- minSize: number | undefined;
77
- minValue: string | undefined;
78
- minWholeDigits: number | undefined;
79
- numericReadings?: Partial<Record<DecimalSeparatorId, NumericReadingConfig>> | undefined;
80
- missingValueCount: number | undefined;
53
+ dateOrderId: DateOrderId | undefined;
81
54
  nullWordsAreEmpty?: boolean | undefined;
82
- nullWordValueCount?: number | undefined;
83
- formats: Record<string, number>;
84
- singleValueCount?: number | undefined;
85
- storageTypeId: StorageTypeId | undefined;
86
- validValueCount: number | undefined;
87
- validValues: Record<string, string>;
88
- valueBinOutliers?: ValueBinOutlierConfig | undefined;
89
- valueBinScaleId?: ValueBinScaleId | undefined;
90
- valueBins?: ValueBinConfig[] | undefined;
91
- voidValueCount: number | undefined;
92
- }
93
- export interface ValueBinOutlierConfig {
94
- aboveCount: number;
95
- belowCount: number;
96
- isEstimated: boolean;
97
- }
98
- export type ValueBinScaleId = 'linear' | 'log';
99
- export interface ValueBinConfig {
100
- count: number;
101
- lowerValue: string;
102
- upperValue: string;
103
55
  }
104
- export type ColumnDerivationId = 'currency' | 'unit';
105
- export type NumericReadingConfig = Pick<ObjectColumnConfig, 'dataSubtypeId' | 'maxDecimals' | 'maxValue' | 'maxWholeDigits' | 'minDecimals' | 'minValue' | 'minWholeDigits' | 'storageTypeId'>;
106
- export type DateReadingConfig = Pick<ObjectColumnConfig, 'formats' | 'maxValue' | 'minValue'>;
107
56
  export type DateOrderId = 'dayFirst' | 'monthFirst';
108
57
  export type DecimalSeparatorId = (typeof DECIMAL_SEPARATOR_IDS)[number];
109
58
  export type StorageTypeId = 'binary' | 'boolean' | 'byte' | 'date' | 'dateTime' | 'dateTimeOffset' | 'decimal' | 'double' | 'int8' | 'int16' | 'int32' | 'int64' | 'object' | 'single' | 'string' | 'time' | 'unknown';
@@ -22,7 +22,6 @@ export interface ReadingConfig {
22
22
  valueDelimiterId: ValueDelimiterId | undefined;
23
23
  }
24
24
  export type DataFormatId = (typeof DATA_FORMAT_CONFIGS)[number]['id'];
25
- export type DataFormatSupportId = 'planned' | 'supported' | 'unsupported';
26
25
  export type RecordDelimiterId = '\n' | '\r' | '\r\n';
27
26
  export type ValueDelimiterId = (typeof VALUE_DELIMITER_CONFIGS)[number]['id'] | '';
28
27
  export interface SampleConfig {
@@ -49,7 +48,7 @@ export type ParsedRecord = {
49
48
  export interface ContentAuditConfig {
50
49
  asAt: number;
51
50
  byteCount: number;
52
- columns: ObjectColumnConfig[];
51
+ columns: ColumnAuditConfig[];
53
52
  commentLineCount: number | undefined;
54
53
  emptyLineCount: number | undefined;
55
54
  invalidFieldLengthCount: number | undefined;
@@ -59,12 +58,59 @@ export interface ContentAuditConfig {
59
58
  recordCount: number;
60
59
  sourceWaitDuration: number | undefined;
61
60
  }
61
+ export interface ColumnAuditConfig extends ObjectColumnConfig {
62
+ inferenceCounts: Record<string, number>;
63
+ leadingInferenceCounts: Record<string, number>;
64
+ numericReadings?: Partial<Record<DecimalSeparatorId, NumericReadingConfig>> | undefined;
65
+ dateReadings?: Partial<Record<DateOrderId, DateReadingConfig>> | undefined;
66
+ isFormatAmbiguous: boolean | undefined;
67
+ nullWordValueCount?: number | undefined;
68
+ validValueCount: number | undefined;
69
+ invalidValueCount: number | undefined;
70
+ blankValueCount: number | undefined;
71
+ missingValueCount: number | undefined;
72
+ isComplete: boolean | undefined;
73
+ validFormats: Record<string, number>;
74
+ validMinSize: number | undefined;
75
+ validMaxSize: number | undefined;
76
+ validMinWholeDigits: number | undefined;
77
+ validMaxWholeDigits: number | undefined;
78
+ validMinDecimals: number | undefined;
79
+ validMaxDecimals: number | undefined;
80
+ validDistinctValueCount: number | undefined;
81
+ validSingleValueCount?: number | undefined;
82
+ invalidDistinctValueCount?: number | undefined;
83
+ validIsUnique: boolean | undefined;
84
+ validFrequentValues: Record<string, number> | undefined;
85
+ validLeastFrequentValues?: Record<string, number> | undefined;
86
+ invalidFrequentValues?: Record<string, number> | undefined;
87
+ invalidValues: Record<string, string>;
88
+ validLowestValues?: [string, number][] | undefined;
89
+ validHighestValues?: [string, number][] | undefined;
90
+ validValueBins?: ValueBinConfig[] | undefined;
91
+ validValueBinScaleId?: ValueBinScaleId | undefined;
92
+ validValueBinOutliers?: ValueBinOutlierConfig | undefined;
93
+ }
94
+ export type NumericReadingConfig = Pick<ColumnAuditConfig, 'dataSubtypeId' | 'validMaxDecimals' | 'validMaxWholeDigits' | 'validMinDecimals' | 'validMinWholeDigits' | 'storageTypeId'>;
95
+ export type DateReadingConfig = Pick<ColumnAuditConfig, 'validFormats'>;
96
+ export interface ValueBinConfig {
97
+ count: number;
98
+ lowerValue: string;
99
+ upperValue: string;
100
+ }
101
+ export type ValueBinScaleId = 'linear' | 'log';
102
+ export interface ValueBinOutlierConfig {
103
+ aboveCount: number;
104
+ belowCount: number;
105
+ isEstimated: boolean;
106
+ }
62
107
  export type ColumnChoiceConfigs = Partial<Record<string, ColumnChoiceConfig>>;
63
108
  export interface ColumnChoiceConfig {
64
109
  dataTypeId: DataTypeId | undefined;
65
110
  dataSubtypeId: DataSubtypeId | undefined;
66
111
  dateOrderId: DateOrderId | undefined;
67
112
  decimalSeparatorId: DecimalSeparatorId | undefined;
113
+ isIgnored: boolean | undefined;
68
114
  nullWordsAreEmpty: boolean | undefined;
69
115
  }
70
116
  type DataTypeConfig = (typeof DATA_TYPE_CONFIGS)[number];
@@ -3,15 +3,13 @@ import { Component } from '../..';
3
3
  import { EngineConnectorActionOptions } from '../engine';
4
4
  import { LocalisedConfig } from '../../../locale';
5
5
  import { ToolConfig } from '../tool';
6
+ import { ColumnAuditConfig, ContentAuditConfig, DataFormatId, ParsedRecord, ReadingConfig, SampleConfig, ValueDelimiterId } from '../../dataView';
6
7
  import { ConnectionConfig, ConnectionDescriptionConfig, ConnectionNodeConfig, ObjectColumnConfig } from '../../connection';
7
8
  import { connectorActionNameSchema, connectorCategoryConfigSchema, connectorConfigSchema, connectorItemTypeConfigSchema, connectorItemTypeIdSchema, connectorUsageConfigSchema, connectorUsageIdSchema } from './connectorConfig.schema';
8
- import { ContentAuditConfig, DataFormatId, ParsedRecord, ReadingConfig, SampleConfig, ValueDelimiterId } from '../../dataView';
9
9
  export { connectorConfigSchema } from './connectorConfig.schema';
10
10
  export { auditObjectContentWithParser, buildPreviewObjectResult, type ObjectFilePreview, retrieveRecordsWithParser } from './objectParsing';
11
11
  export interface ConnectorInterface extends Component {
12
- abortController: AbortController | undefined;
13
12
  readonly config: ConnectorConfig;
14
- readonly toolConfigs: ToolConfig[];
15
13
  abortOperation?(): void;
16
14
  auditObjectContent?(options: AuditObjectContentOptions, chunk: (recordCount: number, byteCount: number) => void): Promise<AuditObjectContentResult>;
17
15
  closeStore?(options: CloseStoreOptions): Promise<void>;
@@ -45,7 +43,7 @@ export interface ColumnAuditor {
45
43
  completeAudit: () => InferenceSummary;
46
44
  }
47
45
  export interface InferenceSummary {
48
- columnConfigs: ObjectColumnConfig[];
46
+ columnConfigs: ColumnAuditConfig[];
49
47
  hasHeaders: boolean;
50
48
  }
51
49
  export type ConnectorConfig = InferOutput<typeof connectorConfigSchema>;
@@ -126,6 +124,7 @@ export interface LoadObjectRecordsResult {
126
124
  }
127
125
  export interface EngineLoadObjectRecordsOptions extends LoadObjectRecordsOptions {
128
126
  hasHeaders: boolean;
127
+ ignoredColumnIndexes: number[];
129
128
  headerValues: (string | null)[] | undefined;
130
129
  sourceConnectionConfig: ConnectionConfig | LocalisedConfig<ConnectionConfig>;
131
130
  sourceOptions: RetrieveRecordsOptions;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dpuse/dpuse-shared",
3
- "version": "1.0.198",
3
+ "version": "1.0.199",
4
4
  "private": false,
5
5
  "description": "Common constants, types and utilities used across DPUse projects.",
6
6
  "license": "MIT",