@dpuse/dpuse-shared 1.0.184 → 1.0.186
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dpuse-shared.es.js +2 -2
- package/dist/dpuse-shared.es.js.map +1 -1
- package/dist/types/src/component/dataView/index.d.ts +16 -77
- package/dist/types/src/component/module/connector/index.d.ts +6 -7
- package/dist/types/src/component/module/connector/objectParsing.d.ts +2 -2
- package/dist/types/src/component/module/tool/delimitedText.d.ts +2 -2
- package/dist/types/src/component/module/tool/parser.d.ts +3 -3
- package/package.json +1 -1
|
@@ -11,13 +11,6 @@ export interface DataViewConfig extends ComponentInstanceConfig {
|
|
|
11
11
|
columnChoiceConfigs: ColumnChoiceConfigs | undefined;
|
|
12
12
|
storeConnectorId: string | undefined;
|
|
13
13
|
}
|
|
14
|
-
export type ColumnChoiceConfigs = Partial<Record<string, ColumnChoiceConfig>>;
|
|
15
|
-
export interface ColumnChoiceConfig {
|
|
16
|
-
dataSubtypeId?: DataSubtypeId | undefined;
|
|
17
|
-
dataTypeId?: DataTypeId | undefined;
|
|
18
|
-
decimalSeparatorId?: DecimalSeparatorId | undefined;
|
|
19
|
-
nullWordsAreEmpty?: boolean | undefined;
|
|
20
|
-
}
|
|
21
14
|
export interface ReadingConfig {
|
|
22
15
|
dataFormatId: DataFormatId;
|
|
23
16
|
encodingConfidenceLevel: number | undefined;
|
|
@@ -38,11 +31,14 @@ export interface ReadingConfig {
|
|
|
38
31
|
skipLinesWithEmptyValues?: boolean | undefined;
|
|
39
32
|
skipLinesWithErrors?: boolean | undefined;
|
|
40
33
|
valueDelimiterOtherCharSeq?: string | undefined;
|
|
41
|
-
valueTrimMethodId?:
|
|
34
|
+
valueTrimMethodId?: 'both' | 'left' | 'right' | 'none' | undefined;
|
|
42
35
|
}
|
|
36
|
+
export type DataFormatId = 'arrow' | 'avro' | 'dtv' | 'json' | 'jsonl' | 'markdown' | 'ods' | 'parquet' | 'sav' | 'sqlite' | 'xlsx' | 'xml' | 'unknown';
|
|
37
|
+
export type RecordDelimiterId = '\n' | '\r' | '\r\n';
|
|
38
|
+
export type ValueDelimiterId = '' | ':' | ',' | '!' | '0x1E' | ';' | ' ' | '\t' | '_' | '0x1F' | '|';
|
|
43
39
|
export interface SampleConfig {
|
|
44
40
|
issues?: SampleIssue[];
|
|
45
|
-
parsedRecords:
|
|
41
|
+
parsedRecords: ParsedRecord[];
|
|
46
42
|
size: number | undefined;
|
|
47
43
|
text: string | undefined;
|
|
48
44
|
}
|
|
@@ -57,15 +53,10 @@ export interface SampleIssue {
|
|
|
57
53
|
value?: string | undefined;
|
|
58
54
|
}
|
|
59
55
|
export type SampleIssueSeverityId = 'error' | 'info' | 'warning';
|
|
60
|
-
export type
|
|
61
|
-
export type ParsingRecord = ParsingResult[];
|
|
62
|
-
interface ParsingResult {
|
|
56
|
+
export type ParsedRecord = {
|
|
63
57
|
value: string | null;
|
|
64
58
|
valueWasQuoted: boolean;
|
|
65
|
-
}
|
|
66
|
-
export type RecordDelimiterId = '\n' | '\r' | '\r\n';
|
|
67
|
-
export type ValueDelimiterId = '' | ':' | ',' | '!' | '0x1E' | ';' | ' ' | '\t' | '_' | '0x1F' | '|';
|
|
68
|
-
type ValueTrimMethodId = 'both' | 'left' | 'right' | 'none';
|
|
59
|
+
}[];
|
|
69
60
|
export interface ContentAuditConfig {
|
|
70
61
|
asAt: number;
|
|
71
62
|
byteCount: number;
|
|
@@ -79,73 +70,21 @@ export interface ContentAuditConfig {
|
|
|
79
70
|
recordCount: number;
|
|
80
71
|
sourceWaitDuration: number | undefined;
|
|
81
72
|
}
|
|
73
|
+
export type ColumnChoiceConfigs = Partial<Record<string, ColumnChoiceConfig>>;
|
|
74
|
+
export interface ColumnChoiceConfig {
|
|
75
|
+
dataSubtypeId?: DataSubtypeId | undefined;
|
|
76
|
+
dataTypeId?: DataTypeId | undefined;
|
|
77
|
+
decimalSeparatorId?: DecimalSeparatorId | undefined;
|
|
78
|
+
nullWordsAreEmpty?: boolean | undefined;
|
|
79
|
+
}
|
|
82
80
|
export type DataTypeId = 'boolean' | 'numeric' | 'string' | 'temporal' | 'unknown';
|
|
83
81
|
export type DataSubtypeId = NumericSubtypeId | StringSubtypeId | TemporalSubtypeId;
|
|
84
82
|
export type NumericSubtypeId = 'bigint' | 'integer' | 'decimal';
|
|
85
|
-
type StringSubtypeId = 'email' | 'ipv4' | 'ipv6' | 'ulid' | 'uuid' | 'url' | 'plain';
|
|
83
|
+
export type StringSubtypeId = 'email' | 'ipv4' | 'ipv6' | 'ulid' | 'uuid' | 'url' | 'plain';
|
|
86
84
|
export type TemporalSubtypeId = 'date' | 'dateTime' | 'time';
|
|
87
85
|
export interface InferenceSummary {
|
|
88
86
|
columnConfigs: ObjectColumnConfig[];
|
|
89
|
-
|
|
90
|
-
}
|
|
91
|
-
export type InferenceRecord = InferenceResult[];
|
|
92
|
-
export type InferenceResult = BooleanInferenceResult | NumericInferenceResult | StringInferenceResult | TemporalInferenceResult | UnknownInferenceResult;
|
|
93
|
-
export interface BooleanInferenceResult {
|
|
94
|
-
dataTypeId: 'boolean';
|
|
95
|
-
dataSubtypeId: undefined;
|
|
96
|
-
inputValue: string;
|
|
97
|
-
inputValueWasQuoted: boolean;
|
|
98
|
-
inferredValue: boolean;
|
|
99
|
-
}
|
|
100
|
-
export type NumericInferenceResult = BigIntInferenceResult | NumberInferenceResult;
|
|
101
|
-
interface BigIntInferenceResult {
|
|
102
|
-
dataTypeId: 'numeric';
|
|
103
|
-
dataSubtypeId: 'bigint';
|
|
104
|
-
format: string;
|
|
105
|
-
inputValue: string;
|
|
106
|
-
inputValueWasQuoted: boolean;
|
|
107
|
-
inferredValue: bigint;
|
|
108
|
-
currencySymbolId: string | undefined;
|
|
109
|
-
decimalPlaces: number;
|
|
110
|
-
signId: NumericSignId;
|
|
111
|
-
unitsId: NumericUnitsId;
|
|
112
|
-
}
|
|
113
|
-
interface NumberInferenceResult {
|
|
114
|
-
dataTypeId: 'numeric';
|
|
115
|
-
dataSubtypeId: 'integer' | 'decimal';
|
|
116
|
-
format: string;
|
|
117
|
-
inputValue: string;
|
|
118
|
-
inputValueWasQuoted: boolean;
|
|
119
|
-
inferredValue: number;
|
|
120
|
-
currencySymbolId: string | undefined;
|
|
121
|
-
decimalPlaces: number;
|
|
122
|
-
signId: NumericSignId;
|
|
123
|
-
unitsId: NumericUnitsId;
|
|
124
|
-
}
|
|
125
|
-
export type NumericSignId = 'negative' | 'zero' | 'positive';
|
|
126
|
-
export type NumericUnitsId = 'currency' | 'percentage' | 'plain';
|
|
127
|
-
export interface StringInferenceResult {
|
|
128
|
-
dataTypeId: 'string';
|
|
129
|
-
dataSubtypeId: StringSubtypeId;
|
|
130
|
-
format: string | undefined;
|
|
131
|
-
inputValue: string;
|
|
132
|
-
inputValueWasQuoted: boolean;
|
|
133
|
-
inferredValue: string;
|
|
134
|
-
}
|
|
135
|
-
export interface TemporalInferenceResult {
|
|
136
|
-
dataTypeId: 'temporal';
|
|
137
|
-
dataSubtypeId: TemporalSubtypeId;
|
|
138
|
-
format: string;
|
|
139
|
-
inputValue: string;
|
|
140
|
-
inputValueWasQuoted: boolean;
|
|
141
|
-
inferredValue: Date;
|
|
142
|
-
}
|
|
143
|
-
interface UnknownInferenceResult {
|
|
144
|
-
dataTypeId: 'unknown';
|
|
145
|
-
dataSubtypeId: undefined;
|
|
146
|
-
inputValue: string | null;
|
|
147
|
-
inputValueWasQuoted: boolean;
|
|
148
|
-
inferredValue: null;
|
|
87
|
+
hasHeaders: boolean;
|
|
149
88
|
}
|
|
150
89
|
export { DATA_FORMAT_ICON_CONFIGS, type DataFormatIconConfig } from './dataFormatIcons';
|
|
151
90
|
export declare const ORDERED_VALUE_DELIMITER_IDS: ValueDelimiterId[];
|
|
@@ -5,7 +5,7 @@ import { LocalisedConfig } from '../../../locale';
|
|
|
5
5
|
import { ToolConfig } from '../tool';
|
|
6
6
|
import { ConnectionConfig, ConnectionDescriptionConfig, ConnectionNodeConfig, ObjectColumnConfig } from '../../connection';
|
|
7
7
|
import { connectorActionNameSchema, connectorCategoryConfigSchema, connectorConfigSchema, connectorItemTypeConfigSchema, connectorItemTypeIdSchema, connectorUsageConfigSchema, connectorUsageIdSchema } from './connectorConfig.schema';
|
|
8
|
-
import { ContentAuditConfig, DataFormatId,
|
|
8
|
+
import { ContentAuditConfig, DataFormatId, InferenceSummary, ParsedRecord, ReadingConfig, SampleConfig, ValueDelimiterId } from '../../dataView';
|
|
9
9
|
export { connectorConfigSchema } from './connectorConfig.schema';
|
|
10
10
|
export { auditObjectContentWithParser, buildPreviewObjectResult, type ObjectFilePreview, retrieveRecordsWithParser } from './objectParsing';
|
|
11
11
|
export interface ConnectorInterface extends Component {
|
|
@@ -29,7 +29,7 @@ export interface ConnectorInterface extends Component {
|
|
|
29
29
|
queryRecords?(options: QueryRecordsOptions): Promise<QueryRecordsResult>;
|
|
30
30
|
removeRecords?(options: RemoveRecordsOptions): Promise<void>;
|
|
31
31
|
retrieveChunks?(options: RetrieveChunksOptions, chunk: (data: Uint8Array) => void, complete: () => void): Promise<void>;
|
|
32
|
-
retrieveRecords?(options: RetrieveRecordsOptions, chunk: (typeId: RecordRetrievalTypeId, records: Record<string, unknown>[] |
|
|
32
|
+
retrieveRecords?(options: RetrieveRecordsOptions, chunk: (typeId: RecordRetrievalTypeId, records: Record<string, unknown>[] | ParsedRecord[]) => void | Promise<void>, // A returned promise holds the read back until it settles.
|
|
33
33
|
complete: (result: RetrieveRecordsSummary) => void): Promise<void>;
|
|
34
34
|
upsertRecords?(options: UpsertRecordsOptions): Promise<void>;
|
|
35
35
|
}
|
|
@@ -38,11 +38,10 @@ export type ConnectorConstructor = new (connectorUtilities: ConnectorUtilities,
|
|
|
38
38
|
export interface ConnectorUtilities {
|
|
39
39
|
constructColumnAuditor: (hasHeaders: boolean | undefined) => ColumnAuditor;
|
|
40
40
|
hasReadableStreamTransferSupport(): boolean;
|
|
41
|
-
|
|
42
|
-
inferDataTypes: (parsedRecords: ParsingRecord[], hasHeaders?: boolean) => InferenceSummary;
|
|
41
|
+
inferDataTypes: (parsedRecords: ParsedRecord[], hasHeaders?: boolean) => InferenceSummary;
|
|
43
42
|
}
|
|
44
43
|
export interface ColumnAuditor {
|
|
45
|
-
auditRecords: (parsedRecords:
|
|
44
|
+
auditRecords: (parsedRecords: ParsedRecord[]) => void;
|
|
46
45
|
completeAudit: () => {
|
|
47
46
|
columnConfigs: ObjectColumnConfig[];
|
|
48
47
|
hasHeaders: boolean;
|
|
@@ -186,11 +185,11 @@ export interface QueryRecordsResult {
|
|
|
186
185
|
}
|
|
187
186
|
export interface RecountPreviewColumnsOptions extends EngineConnectorActionOptions {
|
|
188
187
|
hasHeaders: boolean;
|
|
189
|
-
parsedRecords:
|
|
188
|
+
parsedRecords: ParsedRecord[];
|
|
190
189
|
}
|
|
191
190
|
export interface ClassifyPreviewValuesOptions extends EngineConnectorActionOptions {
|
|
192
191
|
columnConfigs: ObjectColumnConfig[];
|
|
193
|
-
parsedRecords:
|
|
192
|
+
parsedRecords: ParsedRecord[];
|
|
194
193
|
}
|
|
195
194
|
export type PreviewValueStateId = 'blank' | 'invalid' | 'missing' | 'valid';
|
|
196
195
|
export interface RemoveRecordsOptions extends EngineConnectorActionOptions {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { FileTypeResult } from 'file-type';
|
|
2
2
|
import { AuditObjectContentOptions, AuditObjectContentResult, ColumnAuditor, ConnectorUtilities, PreviewObjectResult, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
|
|
3
|
-
import { DataFormatId,
|
|
3
|
+
import { DataFormatId, ParsedRecord, ValueDelimiterId } from '../../dataView';
|
|
4
4
|
import { ToolConfig } from '../tool';
|
|
5
5
|
export interface ObjectFilePreview {
|
|
6
6
|
bytes: Uint8Array;
|
|
@@ -21,7 +21,7 @@ export declare function buildPreviewObjectResult(input: {
|
|
|
21
21
|
valueDelimiterId: ValueDelimiterId | undefined;
|
|
22
22
|
}): Promise<PreviewObjectResult>;
|
|
23
23
|
export declare function retrieveRecordsWithParser(input: {
|
|
24
|
-
chunk: (typeId: RecordRetrievalTypeId, records:
|
|
24
|
+
chunk: (typeId: RecordRetrievalTypeId, records: ParsedRecord[]) => void | Promise<void>;
|
|
25
25
|
options: RetrieveRecordsOptions;
|
|
26
26
|
signal: AbortSignal;
|
|
27
27
|
stream: ReadableStream<Uint8Array>;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
+
import { ParsedRecord, SampleIssue, ValueDelimiterId } from '../../dataView';
|
|
1
2
|
import { ParseTextOptions, ParseTextResult } from '.';
|
|
2
|
-
import { ParsingRecord, SampleIssue, ValueDelimiterId } from '../../dataView';
|
|
3
3
|
export type ParseTextWithDelimiter = (text: string, delimiter: ValueDelimiterId, isComplete: boolean) => Promise<{
|
|
4
4
|
issues: SampleIssue[];
|
|
5
|
-
records:
|
|
5
|
+
records: ParsedRecord[];
|
|
6
6
|
}>;
|
|
7
7
|
/**
|
|
8
8
|
* Reads delimited text for a preview: detects its record delimiter, and which of the candidate value delimiters splits
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { DataFormatId,
|
|
1
|
+
import { DataFormatId, ParsedRecord, RecordDelimiterId, SampleIssue, ValueDelimiterId } from '../../dataView';
|
|
2
2
|
import { RecordRetrievalTypeId, RetrieveRecordsSummary } from '../connector';
|
|
3
3
|
export interface ParserInterface {
|
|
4
|
-
parseStream: (stream: ReadableStream<Uint8Array>, options: ParseStreamOptions, chunk: (typeId: RecordRetrievalTypeId, records:
|
|
4
|
+
parseStream: (stream: ReadableStream<Uint8Array>, options: ParseStreamOptions, chunk: (typeId: RecordRetrievalTypeId, records: ParsedRecord[]) => void | Promise<void>) => Promise<RetrieveRecordsSummary>;
|
|
5
5
|
parseText: (text: string, options: ParseTextOptions) => Promise<ParseTextResult>;
|
|
6
6
|
}
|
|
7
7
|
export interface ParseTextOptions {
|
|
@@ -11,7 +11,7 @@ export interface ParseTextOptions {
|
|
|
11
11
|
}
|
|
12
12
|
export interface ParseTextResult {
|
|
13
13
|
issues: SampleIssue[];
|
|
14
|
-
parsedRecords:
|
|
14
|
+
parsedRecords: ParsedRecord[];
|
|
15
15
|
recordDelimiterId: RecordDelimiterId;
|
|
16
16
|
valueDelimiterId: ValueDelimiterId | undefined;
|
|
17
17
|
}
|