@dpuse/dpuse-shared 1.0.199 → 1.0.202
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -9
- package/dist/dpuse-shared.es.js +435 -290
- package/dist/dpuse-shared.es.js.map +1 -1
- package/dist/types/src/component/connection/index.d.ts +0 -7
- package/dist/types/src/component/dataView/index.d.ts +24 -2
- package/dist/types/src/component/module/connector/index.d.ts +7 -7
- package/dist/types/src/component/module/connector/objectParsing.d.ts +6 -3
- package/dist/types/src/component/module/connector/tableSchema.d.ts +9 -0
- package/package.json +4 -4
|
@@ -45,16 +45,9 @@ export interface ConnectionNodeConfig extends LocalisedConfig<BaseConfig> {
|
|
|
45
45
|
}
|
|
46
46
|
export interface ObjectColumnConfig {
|
|
47
47
|
label: LocaleLabel;
|
|
48
|
-
derivedFromColumnIndex?: number | undefined;
|
|
49
48
|
dataTypeId: DataTypeId;
|
|
50
49
|
dataSubtypeId: DataSubtypeId | undefined;
|
|
51
50
|
storageTypeId: StorageTypeId | undefined;
|
|
52
|
-
decimalSeparatorId: DecimalSeparatorId | undefined;
|
|
53
|
-
dateOrderId: DateOrderId | undefined;
|
|
54
|
-
nullWordsAreEmpty?: boolean | undefined;
|
|
55
51
|
}
|
|
56
|
-
export type DateOrderId = 'dayFirst' | 'monthFirst';
|
|
57
|
-
export type DecimalSeparatorId = (typeof DECIMAL_SEPARATOR_IDS)[number];
|
|
58
52
|
export type StorageTypeId = 'binary' | 'boolean' | 'byte' | 'date' | 'dateTime' | 'dateTimeOffset' | 'decimal' | 'double' | 'int8' | 'int16' | 'int32' | 'int64' | 'object' | 'single' | 'string' | 'time' | 'unknown';
|
|
59
|
-
export declare const DECIMAL_SEPARATOR_IDS: readonly [",", "."];
|
|
60
53
|
export {};
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { FileTypeResult } from 'file-type';
|
|
2
2
|
import { ComponentInstanceConfig } from '..';
|
|
3
|
-
import { ConnectionNodeConfig,
|
|
3
|
+
import { ConnectionNodeConfig, ObjectColumnConfig } from '../connection';
|
|
4
4
|
export interface DataViewConfig extends ComponentInstanceConfig {
|
|
5
5
|
typeId: 'dataView';
|
|
6
6
|
connectionId: string;
|
|
@@ -58,9 +58,18 @@ export interface ContentAuditConfig {
|
|
|
58
58
|
recordCount: number;
|
|
59
59
|
sourceWaitDuration: number | undefined;
|
|
60
60
|
}
|
|
61
|
-
export interface
|
|
61
|
+
export interface ColumnReadingConfig extends ObjectColumnConfig {
|
|
62
|
+
derivedFromColumnIndex?: number | undefined;
|
|
63
|
+
decimalSeparatorId: DecimalSeparatorId | undefined;
|
|
64
|
+
dateOrderId: DateOrderId | undefined;
|
|
65
|
+
nullWordsAreEmpty?: boolean | undefined;
|
|
66
|
+
}
|
|
67
|
+
export type DateOrderId = 'dayFirst' | 'monthFirst';
|
|
68
|
+
export type DecimalSeparatorId = (typeof DECIMAL_SEPARATOR_IDS)[number];
|
|
69
|
+
export interface ColumnAuditConfig extends ColumnReadingConfig {
|
|
62
70
|
inferenceCounts: Record<string, number>;
|
|
63
71
|
leadingInferenceCounts: Record<string, number>;
|
|
72
|
+
isTypeDeclared?: boolean | undefined;
|
|
64
73
|
numericReadings?: Partial<Record<DecimalSeparatorId, NumericReadingConfig>> | undefined;
|
|
65
74
|
dateReadings?: Partial<Record<DateOrderId, DateReadingConfig>> | undefined;
|
|
66
75
|
isFormatAmbiguous: boolean | undefined;
|
|
@@ -91,6 +100,18 @@ export interface ColumnAuditConfig extends ObjectColumnConfig {
|
|
|
91
100
|
validValueBinScaleId?: ValueBinScaleId | undefined;
|
|
92
101
|
validValueBinOutliers?: ValueBinOutlierConfig | undefined;
|
|
93
102
|
}
|
|
103
|
+
export interface DeclaredColumnConfig {
|
|
104
|
+
name: string;
|
|
105
|
+
label: string | undefined;
|
|
106
|
+
dataTypeId: Exclude<DataTypeId, 'unknown'> | undefined;
|
|
107
|
+
dataSubtypeId: DataSubtypeId | undefined;
|
|
108
|
+
}
|
|
109
|
+
export interface DeclaredReadingConfig {
|
|
110
|
+
dataFormatId: DataFormatId;
|
|
111
|
+
encodingId: string | undefined;
|
|
112
|
+
hasHeaders: boolean | undefined;
|
|
113
|
+
valueDelimiterId: ValueDelimiterId | undefined;
|
|
114
|
+
}
|
|
94
115
|
export type NumericReadingConfig = Pick<ColumnAuditConfig, 'dataSubtypeId' | 'validMaxDecimals' | 'validMaxWholeDigits' | 'validMinDecimals' | 'validMinWholeDigits' | 'storageTypeId'>;
|
|
95
116
|
export type DateReadingConfig = Pick<ColumnAuditConfig, 'validFormats'>;
|
|
96
117
|
export interface ValueBinConfig {
|
|
@@ -248,5 +269,6 @@ export declare const DATA_TYPE_CONFIGS: readonly [{
|
|
|
248
269
|
readonly isChoosable: false;
|
|
249
270
|
readonly subtypeIds: readonly [];
|
|
250
271
|
}];
|
|
272
|
+
export declare const DECIMAL_SEPARATOR_IDS: readonly [",", "."];
|
|
251
273
|
export declare function getValueDelimiterCharacter(valueDelimiterId: ValueDelimiterId): string;
|
|
252
274
|
export {};
|
|
@@ -1,15 +1,14 @@
|
|
|
1
1
|
import { InferOutput } from 'valibot';
|
|
2
|
-
import { Component } from '../..';
|
|
3
2
|
import { EngineConnectorActionOptions } from '../engine';
|
|
4
3
|
import { LocalisedConfig } from '../../../locale';
|
|
5
4
|
import { ToolConfig } from '../tool';
|
|
6
|
-
import { ColumnAuditConfig, ContentAuditConfig, DataFormatId, ParsedRecord, ReadingConfig, SampleConfig, ValueDelimiterId } from '../../dataView';
|
|
5
|
+
import { ColumnAuditConfig, ColumnReadingConfig, ContentAuditConfig, DataFormatId, DeclaredColumnConfig, ParsedRecord, ReadingConfig, SampleConfig, ValueDelimiterId } from '../../dataView';
|
|
7
6
|
import { ConnectionConfig, ConnectionDescriptionConfig, ConnectionNodeConfig, ObjectColumnConfig } from '../../connection';
|
|
8
7
|
import { connectorActionNameSchema, connectorCategoryConfigSchema, connectorConfigSchema, connectorItemTypeConfigSchema, connectorItemTypeIdSchema, connectorUsageConfigSchema, connectorUsageIdSchema } from './connectorConfig.schema';
|
|
9
8
|
export { connectorConfigSchema } from './connectorConfig.schema';
|
|
10
9
|
export { auditObjectContentWithParser, buildPreviewObjectResult, type ObjectFilePreview, retrieveRecordsWithParser } from './objectParsing';
|
|
11
|
-
export
|
|
12
|
-
|
|
10
|
+
export { readTableSchema } from './tableSchema';
|
|
11
|
+
export interface ConnectorInterface {
|
|
13
12
|
abortOperation?(): void;
|
|
14
13
|
auditObjectContent?(options: AuditObjectContentOptions, chunk: (recordCount: number, byteCount: number) => void): Promise<AuditObjectContentResult>;
|
|
15
14
|
closeStore?(options: CloseStoreOptions): Promise<void>;
|
|
@@ -34,9 +33,9 @@ export interface ConnectorInterface extends Component {
|
|
|
34
33
|
export type ConnectorActionName = InferOutput<typeof connectorActionNameSchema>;
|
|
35
34
|
export type ConnectorConstructor = new (connectorUtilities: ConnectorUtilities, toolConfigs: ToolConfig[]) => ConnectorInterface;
|
|
36
35
|
export interface ConnectorUtilities {
|
|
37
|
-
constructColumnAuditor: (hasHeaders: boolean | undefined) => ColumnAuditor;
|
|
36
|
+
constructColumnAuditor: (hasHeaders: boolean | undefined, declaredColumnConfigs?: DeclaredColumnConfig[]) => ColumnAuditor;
|
|
38
37
|
hasReadableStreamTransferSupport(): boolean;
|
|
39
|
-
inferDataTypes: (parsedRecords: ParsedRecord[], hasHeaders?: boolean) => InferenceSummary;
|
|
38
|
+
inferDataTypes: (parsedRecords: ParsedRecord[], hasHeaders?: boolean, declaredColumnConfigs?: DeclaredColumnConfig[]) => InferenceSummary;
|
|
40
39
|
}
|
|
41
40
|
export interface ColumnAuditor {
|
|
42
41
|
auditRecords: (parsedRecords: ParsedRecord[]) => void;
|
|
@@ -123,6 +122,7 @@ export interface LoadObjectRecordsResult {
|
|
|
123
122
|
recordCount: number;
|
|
124
123
|
}
|
|
125
124
|
export interface EngineLoadObjectRecordsOptions extends LoadObjectRecordsOptions {
|
|
125
|
+
columnConfigs: ColumnReadingConfig[];
|
|
126
126
|
hasHeaders: boolean;
|
|
127
127
|
ignoredColumnIndexes: number[];
|
|
128
128
|
headerValues: (string | null)[] | undefined;
|
|
@@ -188,7 +188,7 @@ export interface RecountPreviewColumnsOptions extends EngineConnectorActionOptio
|
|
|
188
188
|
parsedRecords: ParsedRecord[];
|
|
189
189
|
}
|
|
190
190
|
export interface ClassifyPreviewValuesOptions extends EngineConnectorActionOptions {
|
|
191
|
-
columnConfigs:
|
|
191
|
+
columnConfigs: ColumnReadingConfig[];
|
|
192
192
|
parsedRecords: ParsedRecord[];
|
|
193
193
|
}
|
|
194
194
|
export type PreviewValueStateId = 'blank' | 'invalid' | 'missing' | 'valid';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { FileTypeResult } from 'file-type';
|
|
2
|
-
import { AuditObjectContentOptions, AuditObjectContentResult,
|
|
3
|
-
import { DataFormatId, ParsedRecord, ValueDelimiterId } from '../../dataView';
|
|
2
|
+
import { AuditObjectContentOptions, AuditObjectContentResult, ConnectorUtilities, PreviewObjectResult, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
|
|
3
|
+
import { DataFormatId, DeclaredColumnConfig, ParsedRecord, ValueDelimiterId } from '../../dataView';
|
|
4
4
|
import { ToolConfig } from '../tool';
|
|
5
5
|
export interface ObjectFilePreview {
|
|
6
6
|
bytes: Uint8Array;
|
|
@@ -13,7 +13,9 @@ export interface ObjectFilePreview {
|
|
|
13
13
|
}
|
|
14
14
|
export declare function buildPreviewObjectResult(input: {
|
|
15
15
|
asAt: number;
|
|
16
|
+
declaredColumnConfigs?: DeclaredColumnConfig[] | undefined;
|
|
16
17
|
filePreview: ObjectFilePreview;
|
|
18
|
+
hasHeaders?: boolean | undefined;
|
|
17
19
|
inferDataTypes: ConnectorUtilities['inferDataTypes'];
|
|
18
20
|
parserId: string | undefined;
|
|
19
21
|
startedAt: number;
|
|
@@ -28,7 +30,8 @@ export declare function retrieveRecordsWithParser(input: {
|
|
|
28
30
|
toolConfigs: ToolConfig[];
|
|
29
31
|
}): Promise<RetrieveRecordsSummary>;
|
|
30
32
|
export declare function auditObjectContentWithParser(input: {
|
|
31
|
-
constructColumnAuditor:
|
|
33
|
+
constructColumnAuditor: ConnectorUtilities['constructColumnAuditor'];
|
|
34
|
+
declaredColumnConfigs?: DeclaredColumnConfig[] | undefined;
|
|
32
35
|
onProgress: (recordCount: number, byteCount: number) => void;
|
|
33
36
|
options: AuditObjectContentOptions;
|
|
34
37
|
signal: AbortSignal;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { DeclaredColumnConfig, DeclaredReadingConfig } from '../../dataView';
|
|
2
|
+
export declare function readTableSchema(resource: {
|
|
3
|
+
dialect?: unknown;
|
|
4
|
+
encoding?: string | undefined;
|
|
5
|
+
schema: unknown;
|
|
6
|
+
}): {
|
|
7
|
+
declaredColumnConfigs: DeclaredColumnConfig[];
|
|
8
|
+
declaredReadingConfig: DeclaredReadingConfig;
|
|
9
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dpuse/dpuse-shared",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.202",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "Common constants, types and utilities used across DPUse projects.",
|
|
6
6
|
"license": "MIT",
|
|
@@ -64,12 +64,12 @@
|
|
|
64
64
|
"valibot": "^1.5.0"
|
|
65
65
|
},
|
|
66
66
|
"devDependencies": {
|
|
67
|
-
"@dpuse/dpuse-development": "^1.0.
|
|
68
|
-
"@dpuse/eslint-config-dpuse": "^1.0.
|
|
67
|
+
"@dpuse/dpuse-development": "^1.0.149",
|
|
68
|
+
"@dpuse/eslint-config-dpuse": "^1.0.150",
|
|
69
69
|
"@types/node": "^26.6.6",
|
|
70
70
|
"@vitest/coverage-v8": "^5.1.0",
|
|
71
71
|
"eslint": "^10.14.0",
|
|
72
|
-
"fallow": "^3.
|
|
72
|
+
"fallow": "^3.42.0",
|
|
73
73
|
"fast-check": "^4.10.2",
|
|
74
74
|
"iconv-lite": "^0.7.3",
|
|
75
75
|
"jschardet": "^4.1.0",
|