@dpuse/dpuse-shared 1.0.198 → 1.0.200

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -44,68 +44,10 @@ export interface ConnectionNodeConfig extends LocalisedConfig<BaseConfig> {
44
44
  typeId: 'folder' | 'object';
45
45
  }
46
46
  export interface ObjectColumnConfig {
47
- blankValueCount: number | undefined;
47
+ label: LocaleLabel;
48
48
  dataTypeId: DataTypeId;
49
49
  dataSubtypeId: DataSubtypeId | undefined;
50
- dateOrderId: DateOrderId | undefined;
51
- dateReadings?: Partial<Record<DateOrderId, DateReadingConfig>> | undefined;
52
- decimalSeparatorId: DecimalSeparatorId | undefined;
53
- derivationId?: ColumnDerivationId | undefined;
54
- derivedFromColumnIndex?: number | undefined;
55
- distinctValueCount: number | undefined;
56
- frequentValues: Record<string, number> | undefined;
57
- highestValues?: [string, number][] | undefined;
58
- inferenceCounts: Record<string, number>;
59
- invalidValueCount: number | undefined;
60
- invalidDistinctValueCount?: number | undefined;
61
- invalidFrequentValues?: Record<string, number> | undefined;
62
- invalidValues: Record<string, string>;
63
- isFormatAmbiguous: boolean | undefined;
64
- isIgnored: boolean | undefined;
65
- isComplete: boolean | undefined;
66
- isUnique: boolean | undefined;
67
- label: LocaleLabel;
68
- leadingInferenceCounts: Record<string, number>;
69
- leastFrequentValues?: Record<string, number> | undefined;
70
- lowestValues?: [string, number][] | undefined;
71
- maxDecimals: number | undefined;
72
- maxSize: number | undefined;
73
- maxValue: string | undefined;
74
- maxWholeDigits: number | undefined;
75
- minDecimals: number | undefined;
76
- minSize: number | undefined;
77
- minValue: string | undefined;
78
- minWholeDigits: number | undefined;
79
- numericReadings?: Partial<Record<DecimalSeparatorId, NumericReadingConfig>> | undefined;
80
- missingValueCount: number | undefined;
81
- nullWordsAreEmpty?: boolean | undefined;
82
- nullWordValueCount?: number | undefined;
83
- formats: Record<string, number>;
84
- singleValueCount?: number | undefined;
85
50
  storageTypeId: StorageTypeId | undefined;
86
- validValueCount: number | undefined;
87
- validValues: Record<string, string>;
88
- valueBinOutliers?: ValueBinOutlierConfig | undefined;
89
- valueBinScaleId?: ValueBinScaleId | undefined;
90
- valueBins?: ValueBinConfig[] | undefined;
91
- voidValueCount: number | undefined;
92
- }
93
- export interface ValueBinOutlierConfig {
94
- aboveCount: number;
95
- belowCount: number;
96
- isEstimated: boolean;
97
- }
98
- export type ValueBinScaleId = 'linear' | 'log';
99
- export interface ValueBinConfig {
100
- count: number;
101
- lowerValue: string;
102
- upperValue: string;
103
51
  }
104
- export type ColumnDerivationId = 'currency' | 'unit';
105
- export type NumericReadingConfig = Pick<ObjectColumnConfig, 'dataSubtypeId' | 'maxDecimals' | 'maxValue' | 'maxWholeDigits' | 'minDecimals' | 'minValue' | 'minWholeDigits' | 'storageTypeId'>;
106
- export type DateReadingConfig = Pick<ObjectColumnConfig, 'formats' | 'maxValue' | 'minValue'>;
107
- export type DateOrderId = 'dayFirst' | 'monthFirst';
108
- export type DecimalSeparatorId = (typeof DECIMAL_SEPARATOR_IDS)[number];
109
52
  export type StorageTypeId = 'binary' | 'boolean' | 'byte' | 'date' | 'dateTime' | 'dateTimeOffset' | 'decimal' | 'double' | 'int8' | 'int16' | 'int32' | 'int64' | 'object' | 'single' | 'string' | 'time' | 'unknown';
110
- export declare const DECIMAL_SEPARATOR_IDS: readonly [",", "."];
111
53
  export {};
@@ -1,6 +1,6 @@
1
1
  import { FileTypeResult } from 'file-type';
2
2
  import { ComponentInstanceConfig } from '..';
3
- import { ConnectionNodeConfig, DateOrderId, DecimalSeparatorId, ObjectColumnConfig } from '../connection';
3
+ import { ConnectionNodeConfig, ObjectColumnConfig } from '../connection';
4
4
  export interface DataViewConfig extends ComponentInstanceConfig {
5
5
  typeId: 'dataView';
6
6
  connectionId: string;
@@ -22,7 +22,6 @@ export interface ReadingConfig {
22
22
  valueDelimiterId: ValueDelimiterId | undefined;
23
23
  }
24
24
  export type DataFormatId = (typeof DATA_FORMAT_CONFIGS)[number]['id'];
25
- export type DataFormatSupportId = 'planned' | 'supported' | 'unsupported';
26
25
  export type RecordDelimiterId = '\n' | '\r' | '\r\n';
27
26
  export type ValueDelimiterId = (typeof VALUE_DELIMITER_CONFIGS)[number]['id'] | '';
28
27
  export interface SampleConfig {
@@ -49,7 +48,7 @@ export type ParsedRecord = {
49
48
  export interface ContentAuditConfig {
50
49
  asAt: number;
51
50
  byteCount: number;
52
- columns: ObjectColumnConfig[];
51
+ columns: ColumnAuditConfig[];
53
52
  commentLineCount: number | undefined;
54
53
  emptyLineCount: number | undefined;
55
54
  invalidFieldLengthCount: number | undefined;
@@ -59,12 +58,67 @@ export interface ContentAuditConfig {
59
58
  recordCount: number;
60
59
  sourceWaitDuration: number | undefined;
61
60
  }
61
+ export interface ColumnReadingConfig extends ObjectColumnConfig {
62
+ derivedFromColumnIndex?: number | undefined;
63
+ decimalSeparatorId: DecimalSeparatorId | undefined;
64
+ dateOrderId: DateOrderId | undefined;
65
+ nullWordsAreEmpty?: boolean | undefined;
66
+ }
67
+ export type DateOrderId = 'dayFirst' | 'monthFirst';
68
+ export type DecimalSeparatorId = (typeof DECIMAL_SEPARATOR_IDS)[number];
69
+ export interface ColumnAuditConfig extends ColumnReadingConfig {
70
+ inferenceCounts: Record<string, number>;
71
+ leadingInferenceCounts: Record<string, number>;
72
+ numericReadings?: Partial<Record<DecimalSeparatorId, NumericReadingConfig>> | undefined;
73
+ dateReadings?: Partial<Record<DateOrderId, DateReadingConfig>> | undefined;
74
+ isFormatAmbiguous: boolean | undefined;
75
+ nullWordValueCount?: number | undefined;
76
+ validValueCount: number | undefined;
77
+ invalidValueCount: number | undefined;
78
+ blankValueCount: number | undefined;
79
+ missingValueCount: number | undefined;
80
+ isComplete: boolean | undefined;
81
+ validFormats: Record<string, number>;
82
+ validMinSize: number | undefined;
83
+ validMaxSize: number | undefined;
84
+ validMinWholeDigits: number | undefined;
85
+ validMaxWholeDigits: number | undefined;
86
+ validMinDecimals: number | undefined;
87
+ validMaxDecimals: number | undefined;
88
+ validDistinctValueCount: number | undefined;
89
+ validSingleValueCount?: number | undefined;
90
+ invalidDistinctValueCount?: number | undefined;
91
+ validIsUnique: boolean | undefined;
92
+ validFrequentValues: Record<string, number> | undefined;
93
+ validLeastFrequentValues?: Record<string, number> | undefined;
94
+ invalidFrequentValues?: Record<string, number> | undefined;
95
+ invalidValues: Record<string, string>;
96
+ validLowestValues?: [string, number][] | undefined;
97
+ validHighestValues?: [string, number][] | undefined;
98
+ validValueBins?: ValueBinConfig[] | undefined;
99
+ validValueBinScaleId?: ValueBinScaleId | undefined;
100
+ validValueBinOutliers?: ValueBinOutlierConfig | undefined;
101
+ }
102
+ export type NumericReadingConfig = Pick<ColumnAuditConfig, 'dataSubtypeId' | 'validMaxDecimals' | 'validMaxWholeDigits' | 'validMinDecimals' | 'validMinWholeDigits' | 'storageTypeId'>;
103
+ export type DateReadingConfig = Pick<ColumnAuditConfig, 'validFormats'>;
104
+ export interface ValueBinConfig {
105
+ count: number;
106
+ lowerValue: string;
107
+ upperValue: string;
108
+ }
109
+ export type ValueBinScaleId = 'linear' | 'log';
110
+ export interface ValueBinOutlierConfig {
111
+ aboveCount: number;
112
+ belowCount: number;
113
+ isEstimated: boolean;
114
+ }
62
115
  export type ColumnChoiceConfigs = Partial<Record<string, ColumnChoiceConfig>>;
63
116
  export interface ColumnChoiceConfig {
64
117
  dataTypeId: DataTypeId | undefined;
65
118
  dataSubtypeId: DataSubtypeId | undefined;
66
119
  dateOrderId: DateOrderId | undefined;
67
120
  decimalSeparatorId: DecimalSeparatorId | undefined;
121
+ isIgnored: boolean | undefined;
68
122
  nullWordsAreEmpty: boolean | undefined;
69
123
  }
70
124
  type DataTypeConfig = (typeof DATA_TYPE_CONFIGS)[number];
@@ -202,5 +256,6 @@ export declare const DATA_TYPE_CONFIGS: readonly [{
202
256
  readonly isChoosable: false;
203
257
  readonly subtypeIds: readonly [];
204
258
  }];
259
+ export declare const DECIMAL_SEPARATOR_IDS: readonly [",", "."];
205
260
  export declare function getValueDelimiterCharacter(valueDelimiterId: ValueDelimiterId): string;
206
261
  export {};
@@ -1,17 +1,13 @@
1
1
  import { InferOutput } from 'valibot';
2
- import { Component } from '../..';
3
2
  import { EngineConnectorActionOptions } from '../engine';
4
3
  import { LocalisedConfig } from '../../../locale';
5
4
  import { ToolConfig } from '../tool';
5
+ import { ColumnAuditConfig, ColumnReadingConfig, ContentAuditConfig, DataFormatId, ParsedRecord, ReadingConfig, SampleConfig, ValueDelimiterId } from '../../dataView';
6
6
  import { ConnectionConfig, ConnectionDescriptionConfig, ConnectionNodeConfig, ObjectColumnConfig } from '../../connection';
7
7
  import { connectorActionNameSchema, connectorCategoryConfigSchema, connectorConfigSchema, connectorItemTypeConfigSchema, connectorItemTypeIdSchema, connectorUsageConfigSchema, connectorUsageIdSchema } from './connectorConfig.schema';
8
- import { ContentAuditConfig, DataFormatId, ParsedRecord, ReadingConfig, SampleConfig, ValueDelimiterId } from '../../dataView';
9
8
  export { connectorConfigSchema } from './connectorConfig.schema';
10
9
  export { auditObjectContentWithParser, buildPreviewObjectResult, type ObjectFilePreview, retrieveRecordsWithParser } from './objectParsing';
11
- export interface ConnectorInterface extends Component {
12
- abortController: AbortController | undefined;
13
- readonly config: ConnectorConfig;
14
- readonly toolConfigs: ToolConfig[];
10
+ export interface ConnectorInterface {
15
11
  abortOperation?(): void;
16
12
  auditObjectContent?(options: AuditObjectContentOptions, chunk: (recordCount: number, byteCount: number) => void): Promise<AuditObjectContentResult>;
17
13
  closeStore?(options: CloseStoreOptions): Promise<void>;
@@ -45,7 +41,7 @@ export interface ColumnAuditor {
45
41
  completeAudit: () => InferenceSummary;
46
42
  }
47
43
  export interface InferenceSummary {
48
- columnConfigs: ObjectColumnConfig[];
44
+ columnConfigs: ColumnAuditConfig[];
49
45
  hasHeaders: boolean;
50
46
  }
51
47
  export type ConnectorConfig = InferOutput<typeof connectorConfigSchema>;
@@ -125,7 +121,9 @@ export interface LoadObjectRecordsResult {
125
121
  recordCount: number;
126
122
  }
127
123
  export interface EngineLoadObjectRecordsOptions extends LoadObjectRecordsOptions {
124
+ columnConfigs: ColumnReadingConfig[];
128
125
  hasHeaders: boolean;
126
+ ignoredColumnIndexes: number[];
129
127
  headerValues: (string | null)[] | undefined;
130
128
  sourceConnectionConfig: ConnectionConfig | LocalisedConfig<ConnectionConfig>;
131
129
  sourceOptions: RetrieveRecordsOptions;
@@ -189,7 +187,7 @@ export interface RecountPreviewColumnsOptions extends EngineConnectorActionOptio
189
187
  parsedRecords: ParsedRecord[];
190
188
  }
191
189
  export interface ClassifyPreviewValuesOptions extends EngineConnectorActionOptions {
192
- columnConfigs: ObjectColumnConfig[];
190
+ columnConfigs: ColumnReadingConfig[];
193
191
  parsedRecords: ParsedRecord[];
194
192
  }
195
193
  export type PreviewValueStateId = 'blank' | 'invalid' | 'missing' | 'valid';
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@dpuse/dpuse-shared",
3
- "version": "1.0.198",
3
+ "version": "1.0.200",
4
4
  "private": false,
5
5
  "description": "Common constants, types and utilities used across DPUse projects.",
6
6
  "license": "MIT",