@dpuse/dpuse-shared 1.0.200 → 1.0.203
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -9
- package/dist/dpuse-shared.es.js +432 -286
- package/dist/dpuse-shared.es.js.map +1 -1
- package/dist/types/src/component/connection/index.d.ts +0 -2
- package/dist/types/src/component/dataView/index.d.ts +17 -2
- package/dist/types/src/component/module/connector/index.d.ts +5 -4
- package/dist/types/src/component/module/connector/objectParsing.d.ts +6 -3
- package/dist/types/src/component/module/connector/tableSchema.d.ts +9 -0
- package/package.json +4 -4
|
@@ -47,7 +47,5 @@ export interface ObjectColumnConfig {
|
|
|
47
47
|
label: LocaleLabel;
|
|
48
48
|
dataTypeId: DataTypeId;
|
|
49
49
|
dataSubtypeId: DataSubtypeId | undefined;
|
|
50
|
-
storageTypeId: StorageTypeId | undefined;
|
|
51
50
|
}
|
|
52
|
-
export type StorageTypeId = 'binary' | 'boolean' | 'byte' | 'date' | 'dateTime' | 'dateTimeOffset' | 'decimal' | 'double' | 'int8' | 'int16' | 'int32' | 'int64' | 'object' | 'single' | 'string' | 'time' | 'unknown';
|
|
53
51
|
export {};
|
|
@@ -17,6 +17,7 @@ export interface ReadingConfig {
|
|
|
17
17
|
encodingId: string | undefined;
|
|
18
18
|
fileType: FileTypeResult | undefined;
|
|
19
19
|
hasHeaders: boolean | undefined;
|
|
20
|
+
isDeclared?: boolean | undefined;
|
|
20
21
|
parserId: string | undefined;
|
|
21
22
|
recordDelimiterId: RecordDelimiterId | undefined;
|
|
22
23
|
valueDelimiterId: ValueDelimiterId | undefined;
|
|
@@ -69,9 +70,10 @@ export type DecimalSeparatorId = (typeof DECIMAL_SEPARATOR_IDS)[number];
|
|
|
69
70
|
export interface ColumnAuditConfig extends ColumnReadingConfig {
|
|
70
71
|
inferenceCounts: Record<string, number>;
|
|
71
72
|
leadingInferenceCounts: Record<string, number>;
|
|
73
|
+
isTypeDeclared?: boolean | undefined;
|
|
72
74
|
numericReadings?: Partial<Record<DecimalSeparatorId, NumericReadingConfig>> | undefined;
|
|
73
75
|
dateReadings?: Partial<Record<DateOrderId, DateReadingConfig>> | undefined;
|
|
74
|
-
|
|
76
|
+
formatIssueIds: FormatIssueId[] | undefined;
|
|
75
77
|
nullWordValueCount?: number | undefined;
|
|
76
78
|
validValueCount: number | undefined;
|
|
77
79
|
invalidValueCount: number | undefined;
|
|
@@ -99,8 +101,21 @@ export interface ColumnAuditConfig extends ColumnReadingConfig {
|
|
|
99
101
|
validValueBinScaleId?: ValueBinScaleId | undefined;
|
|
100
102
|
validValueBinOutliers?: ValueBinOutlierConfig | undefined;
|
|
101
103
|
}
|
|
102
|
-
export
|
|
104
|
+
export interface DeclaredColumnConfig {
|
|
105
|
+
name: string;
|
|
106
|
+
label: string | undefined;
|
|
107
|
+
dataTypeId: Exclude<DataTypeId, 'unknown'> | undefined;
|
|
108
|
+
dataSubtypeId: DataSubtypeId | undefined;
|
|
109
|
+
}
|
|
110
|
+
export interface DeclaredReadingConfig {
|
|
111
|
+
dataFormatId: DataFormatId;
|
|
112
|
+
encodingId: string | undefined;
|
|
113
|
+
hasHeaders: boolean | undefined;
|
|
114
|
+
valueDelimiterId: ValueDelimiterId | undefined;
|
|
115
|
+
}
|
|
116
|
+
export type NumericReadingConfig = Pick<ColumnAuditConfig, 'dataSubtypeId' | 'validMaxDecimals' | 'validMaxWholeDigits' | 'validMinDecimals' | 'validMinWholeDigits'>;
|
|
103
117
|
export type DateReadingConfig = Pick<ColumnAuditConfig, 'validFormats'>;
|
|
118
|
+
export type FormatIssueId = 'booleanPairMixed' | 'currencyMixed' | 'dateOrderMixed' | 'dateOrderUnproven' | 'decimalSeparatorMixed' | 'decimalSeparatorUnproven' | 'decimalSeparatorWeaklyProven' | 'percentageMixed' | 'scientificNotationMixed';
|
|
104
119
|
export interface ValueBinConfig {
|
|
105
120
|
count: number;
|
|
106
121
|
lowerValue: string;
|
|
@@ -2,11 +2,10 @@ import { InferOutput } from 'valibot';
|
|
|
2
2
|
import { EngineConnectorActionOptions } from '../engine';
|
|
3
3
|
import { LocalisedConfig } from '../../../locale';
|
|
4
4
|
import { ToolConfig } from '../tool';
|
|
5
|
-
import { ColumnAuditConfig, ColumnReadingConfig, ContentAuditConfig, DataFormatId, ParsedRecord, ReadingConfig, SampleConfig, ValueDelimiterId } from '../../dataView';
|
|
5
|
+
import { ColumnAuditConfig, ColumnReadingConfig, ContentAuditConfig, DataFormatId, DeclaredColumnConfig, ParsedRecord, ReadingConfig, SampleConfig, ValueDelimiterId } from '../../dataView';
|
|
6
6
|
import { ConnectionConfig, ConnectionDescriptionConfig, ConnectionNodeConfig, ObjectColumnConfig } from '../../connection';
|
|
7
7
|
import { connectorActionNameSchema, connectorCategoryConfigSchema, connectorConfigSchema, connectorItemTypeConfigSchema, connectorItemTypeIdSchema, connectorUsageConfigSchema, connectorUsageIdSchema } from './connectorConfig.schema';
|
|
8
8
|
export { connectorConfigSchema } from './connectorConfig.schema';
|
|
9
|
-
export { auditObjectContentWithParser, buildPreviewObjectResult, type ObjectFilePreview, retrieveRecordsWithParser } from './objectParsing';
|
|
10
9
|
export interface ConnectorInterface {
|
|
11
10
|
abortOperation?(): void;
|
|
12
11
|
auditObjectContent?(options: AuditObjectContentOptions, chunk: (recordCount: number, byteCount: number) => void): Promise<AuditObjectContentResult>;
|
|
@@ -32,9 +31,9 @@ export interface ConnectorInterface {
|
|
|
32
31
|
export type ConnectorActionName = InferOutput<typeof connectorActionNameSchema>;
|
|
33
32
|
export type ConnectorConstructor = new (connectorUtilities: ConnectorUtilities, toolConfigs: ToolConfig[]) => ConnectorInterface;
|
|
34
33
|
export interface ConnectorUtilities {
|
|
35
|
-
constructColumnAuditor: (hasHeaders: boolean | undefined) => ColumnAuditor;
|
|
34
|
+
constructColumnAuditor: (hasHeaders: boolean | undefined, declaredColumnConfigs?: DeclaredColumnConfig[]) => ColumnAuditor;
|
|
36
35
|
hasReadableStreamTransferSupport(): boolean;
|
|
37
|
-
inferDataTypes: (parsedRecords: ParsedRecord[], hasHeaders?: boolean) => InferenceSummary;
|
|
36
|
+
inferDataTypes: (parsedRecords: ParsedRecord[], hasHeaders?: boolean, declaredColumnConfigs?: DeclaredColumnConfig[]) => InferenceSummary;
|
|
38
37
|
}
|
|
39
38
|
export interface ColumnAuditor {
|
|
40
39
|
auditRecords: (parsedRecords: ParsedRecord[]) => void;
|
|
@@ -229,6 +228,8 @@ export interface UpsertRecordsOptions extends EngineConnectorActionOptions {
|
|
|
229
228
|
path: string;
|
|
230
229
|
}
|
|
231
230
|
export type ConnectorUsageId = InferOutput<typeof connectorUsageIdSchema>;
|
|
231
|
+
export { auditObjectContentWithParser, buildPreviewObjectResult, type ObjectFilePreview, retrieveRecordsWithParser } from './objectParsing';
|
|
232
|
+
export { readTableSchema } from './tableSchema';
|
|
232
233
|
export declare function determineConnectorUsageId(actionNames: ConnectorActionName[]): ConnectorUsageId;
|
|
233
234
|
export declare const constructConnectorCategoryConfig: (id: string, localeId?: import('../../../locale').LocaleId) => LocalisedConfig<ConnectorCategoryConfig>;
|
|
234
235
|
export declare const constructConnectorItemTypeConfig: (id: string, localeId?: import('../../../locale').LocaleId) => LocalisedConfig<ConnectorItemTypeConfig>;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { FileTypeResult } from 'file-type';
|
|
2
|
-
import { AuditObjectContentOptions, AuditObjectContentResult,
|
|
3
|
-
import { DataFormatId, ParsedRecord, ValueDelimiterId } from '../../dataView';
|
|
2
|
+
import { AuditObjectContentOptions, AuditObjectContentResult, ConnectorUtilities, PreviewObjectResult, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
|
|
3
|
+
import { DataFormatId, DeclaredColumnConfig, DeclaredReadingConfig, ParsedRecord, ValueDelimiterId } from '../../dataView';
|
|
4
4
|
import { ToolConfig } from '../tool';
|
|
5
5
|
export interface ObjectFilePreview {
|
|
6
6
|
bytes: Uint8Array;
|
|
@@ -13,6 +13,8 @@ export interface ObjectFilePreview {
|
|
|
13
13
|
}
|
|
14
14
|
export declare function buildPreviewObjectResult(input: {
|
|
15
15
|
asAt: number;
|
|
16
|
+
declaredColumnConfigs?: DeclaredColumnConfig[] | undefined;
|
|
17
|
+
declaredReadingConfig?: DeclaredReadingConfig | undefined;
|
|
16
18
|
filePreview: ObjectFilePreview;
|
|
17
19
|
inferDataTypes: ConnectorUtilities['inferDataTypes'];
|
|
18
20
|
parserId: string | undefined;
|
|
@@ -28,7 +30,8 @@ export declare function retrieveRecordsWithParser(input: {
|
|
|
28
30
|
toolConfigs: ToolConfig[];
|
|
29
31
|
}): Promise<RetrieveRecordsSummary>;
|
|
30
32
|
export declare function auditObjectContentWithParser(input: {
|
|
31
|
-
constructColumnAuditor:
|
|
33
|
+
constructColumnAuditor: ConnectorUtilities['constructColumnAuditor'];
|
|
34
|
+
declaredColumnConfigs?: DeclaredColumnConfig[] | undefined;
|
|
32
35
|
onProgress: (recordCount: number, byteCount: number) => void;
|
|
33
36
|
options: AuditObjectContentOptions;
|
|
34
37
|
signal: AbortSignal;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { DeclaredColumnConfig, DeclaredReadingConfig } from '../../dataView';
|
|
2
|
+
export declare function readTableSchema(resource: {
|
|
3
|
+
dialect?: unknown;
|
|
4
|
+
encoding?: string | undefined;
|
|
5
|
+
schema: unknown;
|
|
6
|
+
}): {
|
|
7
|
+
declaredColumnConfigs: DeclaredColumnConfig[];
|
|
8
|
+
declaredReadingConfig: DeclaredReadingConfig;
|
|
9
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dpuse/dpuse-shared",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.203",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "Common constants, types and utilities used across DPUse projects.",
|
|
6
6
|
"license": "MIT",
|
|
@@ -64,12 +64,12 @@
|
|
|
64
64
|
"valibot": "^1.5.0"
|
|
65
65
|
},
|
|
66
66
|
"devDependencies": {
|
|
67
|
-
"@dpuse/dpuse-development": "^1.0.
|
|
68
|
-
"@dpuse/eslint-config-dpuse": "^1.0.
|
|
67
|
+
"@dpuse/dpuse-development": "^1.0.149",
|
|
68
|
+
"@dpuse/eslint-config-dpuse": "^1.0.150",
|
|
69
69
|
"@types/node": "^26.6.6",
|
|
70
70
|
"@vitest/coverage-v8": "^5.1.0",
|
|
71
71
|
"eslint": "^10.14.0",
|
|
72
|
-
"fallow": "^3.
|
|
72
|
+
"fallow": "^3.42.0",
|
|
73
73
|
"fast-check": "^4.10.2",
|
|
74
74
|
"iconv-lite": "^0.7.3",
|
|
75
75
|
"jschardet": "^4.1.0",
|