@dpuse/dpuse-shared 1.0.200 → 1.0.202
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -9
- package/dist/dpuse-shared.es.js +429 -286
- package/dist/dpuse-shared.es.js.map +1 -1
- package/dist/types/src/component/dataView/index.d.ts +13 -0
- package/dist/types/src/component/module/connector/index.d.ts +4 -3
- package/dist/types/src/component/module/connector/objectParsing.d.ts +6 -3
- package/dist/types/src/component/module/connector/tableSchema.d.ts +9 -0
- package/package.json +4 -4
|
@@ -69,6 +69,7 @@ export type DecimalSeparatorId = (typeof DECIMAL_SEPARATOR_IDS)[number];
|
|
|
69
69
|
export interface ColumnAuditConfig extends ColumnReadingConfig {
|
|
70
70
|
inferenceCounts: Record<string, number>;
|
|
71
71
|
leadingInferenceCounts: Record<string, number>;
|
|
72
|
+
isTypeDeclared?: boolean | undefined;
|
|
72
73
|
numericReadings?: Partial<Record<DecimalSeparatorId, NumericReadingConfig>> | undefined;
|
|
73
74
|
dateReadings?: Partial<Record<DateOrderId, DateReadingConfig>> | undefined;
|
|
74
75
|
isFormatAmbiguous: boolean | undefined;
|
|
@@ -99,6 +100,18 @@ export interface ColumnAuditConfig extends ColumnReadingConfig {
|
|
|
99
100
|
validValueBinScaleId?: ValueBinScaleId | undefined;
|
|
100
101
|
validValueBinOutliers?: ValueBinOutlierConfig | undefined;
|
|
101
102
|
}
|
|
103
|
+
export interface DeclaredColumnConfig {
|
|
104
|
+
name: string;
|
|
105
|
+
label: string | undefined;
|
|
106
|
+
dataTypeId: Exclude<DataTypeId, 'unknown'> | undefined;
|
|
107
|
+
dataSubtypeId: DataSubtypeId | undefined;
|
|
108
|
+
}
|
|
109
|
+
export interface DeclaredReadingConfig {
|
|
110
|
+
dataFormatId: DataFormatId;
|
|
111
|
+
encodingId: string | undefined;
|
|
112
|
+
hasHeaders: boolean | undefined;
|
|
113
|
+
valueDelimiterId: ValueDelimiterId | undefined;
|
|
114
|
+
}
|
|
102
115
|
export type NumericReadingConfig = Pick<ColumnAuditConfig, 'dataSubtypeId' | 'validMaxDecimals' | 'validMaxWholeDigits' | 'validMinDecimals' | 'validMinWholeDigits' | 'storageTypeId'>;
|
|
103
116
|
export type DateReadingConfig = Pick<ColumnAuditConfig, 'validFormats'>;
|
|
104
117
|
export interface ValueBinConfig {
|
|
@@ -2,11 +2,12 @@ import { InferOutput } from 'valibot';
|
|
|
2
2
|
import { EngineConnectorActionOptions } from '../engine';
|
|
3
3
|
import { LocalisedConfig } from '../../../locale';
|
|
4
4
|
import { ToolConfig } from '../tool';
|
|
5
|
-
import { ColumnAuditConfig, ColumnReadingConfig, ContentAuditConfig, DataFormatId, ParsedRecord, ReadingConfig, SampleConfig, ValueDelimiterId } from '../../dataView';
|
|
5
|
+
import { ColumnAuditConfig, ColumnReadingConfig, ContentAuditConfig, DataFormatId, DeclaredColumnConfig, ParsedRecord, ReadingConfig, SampleConfig, ValueDelimiterId } from '../../dataView';
|
|
6
6
|
import { ConnectionConfig, ConnectionDescriptionConfig, ConnectionNodeConfig, ObjectColumnConfig } from '../../connection';
|
|
7
7
|
import { connectorActionNameSchema, connectorCategoryConfigSchema, connectorConfigSchema, connectorItemTypeConfigSchema, connectorItemTypeIdSchema, connectorUsageConfigSchema, connectorUsageIdSchema } from './connectorConfig.schema';
|
|
8
8
|
export { connectorConfigSchema } from './connectorConfig.schema';
|
|
9
9
|
export { auditObjectContentWithParser, buildPreviewObjectResult, type ObjectFilePreview, retrieveRecordsWithParser } from './objectParsing';
|
|
10
|
+
export { readTableSchema } from './tableSchema';
|
|
10
11
|
export interface ConnectorInterface {
|
|
11
12
|
abortOperation?(): void;
|
|
12
13
|
auditObjectContent?(options: AuditObjectContentOptions, chunk: (recordCount: number, byteCount: number) => void): Promise<AuditObjectContentResult>;
|
|
@@ -32,9 +33,9 @@ export interface ConnectorInterface {
|
|
|
32
33
|
export type ConnectorActionName = InferOutput<typeof connectorActionNameSchema>;
|
|
33
34
|
export type ConnectorConstructor = new (connectorUtilities: ConnectorUtilities, toolConfigs: ToolConfig[]) => ConnectorInterface;
|
|
34
35
|
export interface ConnectorUtilities {
|
|
35
|
-
constructColumnAuditor: (hasHeaders: boolean | undefined) => ColumnAuditor;
|
|
36
|
+
constructColumnAuditor: (hasHeaders: boolean | undefined, declaredColumnConfigs?: DeclaredColumnConfig[]) => ColumnAuditor;
|
|
36
37
|
hasReadableStreamTransferSupport(): boolean;
|
|
37
|
-
inferDataTypes: (parsedRecords: ParsedRecord[], hasHeaders?: boolean) => InferenceSummary;
|
|
38
|
+
inferDataTypes: (parsedRecords: ParsedRecord[], hasHeaders?: boolean, declaredColumnConfigs?: DeclaredColumnConfig[]) => InferenceSummary;
|
|
38
39
|
}
|
|
39
40
|
export interface ColumnAuditor {
|
|
40
41
|
auditRecords: (parsedRecords: ParsedRecord[]) => void;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { FileTypeResult } from 'file-type';
|
|
2
|
-
import { AuditObjectContentOptions, AuditObjectContentResult,
|
|
3
|
-
import { DataFormatId, ParsedRecord, ValueDelimiterId } from '../../dataView';
|
|
2
|
+
import { AuditObjectContentOptions, AuditObjectContentResult, ConnectorUtilities, PreviewObjectResult, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
|
|
3
|
+
import { DataFormatId, DeclaredColumnConfig, ParsedRecord, ValueDelimiterId } from '../../dataView';
|
|
4
4
|
import { ToolConfig } from '../tool';
|
|
5
5
|
export interface ObjectFilePreview {
|
|
6
6
|
bytes: Uint8Array;
|
|
@@ -13,7 +13,9 @@ export interface ObjectFilePreview {
|
|
|
13
13
|
}
|
|
14
14
|
export declare function buildPreviewObjectResult(input: {
|
|
15
15
|
asAt: number;
|
|
16
|
+
declaredColumnConfigs?: DeclaredColumnConfig[] | undefined;
|
|
16
17
|
filePreview: ObjectFilePreview;
|
|
18
|
+
hasHeaders?: boolean | undefined;
|
|
17
19
|
inferDataTypes: ConnectorUtilities['inferDataTypes'];
|
|
18
20
|
parserId: string | undefined;
|
|
19
21
|
startedAt: number;
|
|
@@ -28,7 +30,8 @@ export declare function retrieveRecordsWithParser(input: {
|
|
|
28
30
|
toolConfigs: ToolConfig[];
|
|
29
31
|
}): Promise<RetrieveRecordsSummary>;
|
|
30
32
|
export declare function auditObjectContentWithParser(input: {
|
|
31
|
-
constructColumnAuditor:
|
|
33
|
+
constructColumnAuditor: ConnectorUtilities['constructColumnAuditor'];
|
|
34
|
+
declaredColumnConfigs?: DeclaredColumnConfig[] | undefined;
|
|
32
35
|
onProgress: (recordCount: number, byteCount: number) => void;
|
|
33
36
|
options: AuditObjectContentOptions;
|
|
34
37
|
signal: AbortSignal;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { DeclaredColumnConfig, DeclaredReadingConfig } from '../../dataView';
|
|
2
|
+
export declare function readTableSchema(resource: {
|
|
3
|
+
dialect?: unknown;
|
|
4
|
+
encoding?: string | undefined;
|
|
5
|
+
schema: unknown;
|
|
6
|
+
}): {
|
|
7
|
+
declaredColumnConfigs: DeclaredColumnConfig[];
|
|
8
|
+
declaredReadingConfig: DeclaredReadingConfig;
|
|
9
|
+
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dpuse/dpuse-shared",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.202",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "Common constants, types and utilities used across DPUse projects.",
|
|
6
6
|
"license": "MIT",
|
|
@@ -64,12 +64,12 @@
|
|
|
64
64
|
"valibot": "^1.5.0"
|
|
65
65
|
},
|
|
66
66
|
"devDependencies": {
|
|
67
|
-
"@dpuse/dpuse-development": "^1.0.
|
|
68
|
-
"@dpuse/eslint-config-dpuse": "^1.0.
|
|
67
|
+
"@dpuse/dpuse-development": "^1.0.149",
|
|
68
|
+
"@dpuse/eslint-config-dpuse": "^1.0.150",
|
|
69
69
|
"@types/node": "^26.6.6",
|
|
70
70
|
"@vitest/coverage-v8": "^5.1.0",
|
|
71
71
|
"eslint": "^10.14.0",
|
|
72
|
-
"fallow": "^3.
|
|
72
|
+
"fallow": "^3.42.0",
|
|
73
73
|
"fast-check": "^4.10.2",
|
|
74
74
|
"iconv-lite": "^0.7.3",
|
|
75
75
|
"jschardet": "^4.1.0",
|