@dpuse/dpuse-shared 1.0.184 → 1.0.185
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dpuse-shared.es.js.map +1 -1
- package/dist/types/src/component/dataView/index.d.ts +14 -15
- package/dist/types/src/component/module/connector/index.d.ts +7 -7
- package/dist/types/src/component/module/connector/objectParsing.d.ts +2 -2
- package/dist/types/src/component/module/tool/delimitedText.d.ts +2 -2
- package/dist/types/src/component/module/tool/parser.d.ts +3 -3
- package/package.json +1 -1
|
@@ -11,13 +11,6 @@ export interface DataViewConfig extends ComponentInstanceConfig {
|
|
|
11
11
|
columnChoiceConfigs: ColumnChoiceConfigs | undefined;
|
|
12
12
|
storeConnectorId: string | undefined;
|
|
13
13
|
}
|
|
14
|
-
export type ColumnChoiceConfigs = Partial<Record<string, ColumnChoiceConfig>>;
|
|
15
|
-
export interface ColumnChoiceConfig {
|
|
16
|
-
dataSubtypeId?: DataSubtypeId | undefined;
|
|
17
|
-
dataTypeId?: DataTypeId | undefined;
|
|
18
|
-
decimalSeparatorId?: DecimalSeparatorId | undefined;
|
|
19
|
-
nullWordsAreEmpty?: boolean | undefined;
|
|
20
|
-
}
|
|
21
14
|
export interface ReadingConfig {
|
|
22
15
|
dataFormatId: DataFormatId;
|
|
23
16
|
encodingConfidenceLevel: number | undefined;
|
|
@@ -38,11 +31,21 @@ export interface ReadingConfig {
|
|
|
38
31
|
skipLinesWithEmptyValues?: boolean | undefined;
|
|
39
32
|
skipLinesWithErrors?: boolean | undefined;
|
|
40
33
|
valueDelimiterOtherCharSeq?: string | undefined;
|
|
41
|
-
valueTrimMethodId?:
|
|
34
|
+
valueTrimMethodId?: 'both' | 'left' | 'right' | 'none' | undefined;
|
|
35
|
+
}
|
|
36
|
+
export type DataFormatId = 'arrow' | 'avro' | 'dtv' | 'json' | 'jsonl' | 'markdown' | 'ods' | 'parquet' | 'sav' | 'sqlite' | 'xlsx' | 'xml' | 'unknown';
|
|
37
|
+
export type RecordDelimiterId = '\n' | '\r' | '\r\n';
|
|
38
|
+
export type ValueDelimiterId = '' | ':' | ',' | '!' | '0x1E' | ';' | ' ' | '\t' | '_' | '0x1F' | '|';
|
|
39
|
+
export type ColumnChoiceConfigs = Partial<Record<string, ColumnChoiceConfig>>;
|
|
40
|
+
export interface ColumnChoiceConfig {
|
|
41
|
+
dataSubtypeId?: DataSubtypeId | undefined;
|
|
42
|
+
dataTypeId?: DataTypeId | undefined;
|
|
43
|
+
decimalSeparatorId?: DecimalSeparatorId | undefined;
|
|
44
|
+
nullWordsAreEmpty?: boolean | undefined;
|
|
42
45
|
}
|
|
43
46
|
export interface SampleConfig {
|
|
44
47
|
issues?: SampleIssue[];
|
|
45
|
-
parsedRecords:
|
|
48
|
+
parsedRecords: ParsedRecord[];
|
|
46
49
|
size: number | undefined;
|
|
47
50
|
text: string | undefined;
|
|
48
51
|
}
|
|
@@ -57,15 +60,11 @@ export interface SampleIssue {
|
|
|
57
60
|
value?: string | undefined;
|
|
58
61
|
}
|
|
59
62
|
export type SampleIssueSeverityId = 'error' | 'info' | 'warning';
|
|
60
|
-
export type
|
|
61
|
-
|
|
62
|
-
interface ParsingResult {
|
|
63
|
+
export type ParsedRecord = ParsedValue[];
|
|
64
|
+
interface ParsedValue {
|
|
63
65
|
value: string | null;
|
|
64
66
|
valueWasQuoted: boolean;
|
|
65
67
|
}
|
|
66
|
-
export type RecordDelimiterId = '\n' | '\r' | '\r\n';
|
|
67
|
-
export type ValueDelimiterId = '' | ':' | ',' | '!' | '0x1E' | ';' | ' ' | '\t' | '_' | '0x1F' | '|';
|
|
68
|
-
type ValueTrimMethodId = 'both' | 'left' | 'right' | 'none';
|
|
69
68
|
export interface ContentAuditConfig {
|
|
70
69
|
asAt: number;
|
|
71
70
|
byteCount: number;
|
|
@@ -5,7 +5,7 @@ import { LocalisedConfig } from '../../../locale';
|
|
|
5
5
|
import { ToolConfig } from '../tool';
|
|
6
6
|
import { ConnectionConfig, ConnectionDescriptionConfig, ConnectionNodeConfig, ObjectColumnConfig } from '../../connection';
|
|
7
7
|
import { connectorActionNameSchema, connectorCategoryConfigSchema, connectorConfigSchema, connectorItemTypeConfigSchema, connectorItemTypeIdSchema, connectorUsageConfigSchema, connectorUsageIdSchema } from './connectorConfig.schema';
|
|
8
|
-
import { ContentAuditConfig, DataFormatId, InferenceRecord, InferenceSummary,
|
|
8
|
+
import { ContentAuditConfig, DataFormatId, InferenceRecord, InferenceSummary, ParsedRecord, ReadingConfig, SampleConfig, ValueDelimiterId } from '../../dataView';
|
|
9
9
|
export { connectorConfigSchema } from './connectorConfig.schema';
|
|
10
10
|
export { auditObjectContentWithParser, buildPreviewObjectResult, type ObjectFilePreview, retrieveRecordsWithParser } from './objectParsing';
|
|
11
11
|
export interface ConnectorInterface extends Component {
|
|
@@ -29,7 +29,7 @@ export interface ConnectorInterface extends Component {
|
|
|
29
29
|
queryRecords?(options: QueryRecordsOptions): Promise<QueryRecordsResult>;
|
|
30
30
|
removeRecords?(options: RemoveRecordsOptions): Promise<void>;
|
|
31
31
|
retrieveChunks?(options: RetrieveChunksOptions, chunk: (data: Uint8Array) => void, complete: () => void): Promise<void>;
|
|
32
|
-
retrieveRecords?(options: RetrieveRecordsOptions, chunk: (typeId: RecordRetrievalTypeId, records: Record<string, unknown>[] |
|
|
32
|
+
retrieveRecords?(options: RetrieveRecordsOptions, chunk: (typeId: RecordRetrievalTypeId, records: Record<string, unknown>[] | ParsedRecord[]) => void | Promise<void>, // A returned promise holds the read back until it settles.
|
|
33
33
|
complete: (result: RetrieveRecordsSummary) => void): Promise<void>;
|
|
34
34
|
upsertRecords?(options: UpsertRecordsOptions): Promise<void>;
|
|
35
35
|
}
|
|
@@ -38,11 +38,11 @@ export type ConnectorConstructor = new (connectorUtilities: ConnectorUtilities,
|
|
|
38
38
|
export interface ConnectorUtilities {
|
|
39
39
|
constructColumnAuditor: (hasHeaders: boolean | undefined) => ColumnAuditor;
|
|
40
40
|
hasReadableStreamTransferSupport(): boolean;
|
|
41
|
-
inferValues: (parsedRecord:
|
|
42
|
-
inferDataTypes: (parsedRecords:
|
|
41
|
+
inferValues: (parsedRecord: ParsedRecord, columnConfigs: ObjectColumnConfig[], hasLeadingRecord: boolean) => InferenceRecord;
|
|
42
|
+
inferDataTypes: (parsedRecords: ParsedRecord[], hasHeaders?: boolean) => InferenceSummary;
|
|
43
43
|
}
|
|
44
44
|
export interface ColumnAuditor {
|
|
45
|
-
auditRecords: (parsedRecords:
|
|
45
|
+
auditRecords: (parsedRecords: ParsedRecord[]) => void;
|
|
46
46
|
completeAudit: () => {
|
|
47
47
|
columnConfigs: ObjectColumnConfig[];
|
|
48
48
|
hasHeaders: boolean;
|
|
@@ -186,11 +186,11 @@ export interface QueryRecordsResult {
|
|
|
186
186
|
}
|
|
187
187
|
export interface RecountPreviewColumnsOptions extends EngineConnectorActionOptions {
|
|
188
188
|
hasHeaders: boolean;
|
|
189
|
-
parsedRecords:
|
|
189
|
+
parsedRecords: ParsedRecord[];
|
|
190
190
|
}
|
|
191
191
|
export interface ClassifyPreviewValuesOptions extends EngineConnectorActionOptions {
|
|
192
192
|
columnConfigs: ObjectColumnConfig[];
|
|
193
|
-
parsedRecords:
|
|
193
|
+
parsedRecords: ParsedRecord[];
|
|
194
194
|
}
|
|
195
195
|
export type PreviewValueStateId = 'blank' | 'invalid' | 'missing' | 'valid';
|
|
196
196
|
export interface RemoveRecordsOptions extends EngineConnectorActionOptions {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { FileTypeResult } from 'file-type';
|
|
2
2
|
import { AuditObjectContentOptions, AuditObjectContentResult, ColumnAuditor, ConnectorUtilities, PreviewObjectResult, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
|
|
3
|
-
import { DataFormatId,
|
|
3
|
+
import { DataFormatId, ParsedRecord, ValueDelimiterId } from '../../dataView';
|
|
4
4
|
import { ToolConfig } from '../tool';
|
|
5
5
|
export interface ObjectFilePreview {
|
|
6
6
|
bytes: Uint8Array;
|
|
@@ -21,7 +21,7 @@ export declare function buildPreviewObjectResult(input: {
|
|
|
21
21
|
valueDelimiterId: ValueDelimiterId | undefined;
|
|
22
22
|
}): Promise<PreviewObjectResult>;
|
|
23
23
|
export declare function retrieveRecordsWithParser(input: {
|
|
24
|
-
chunk: (typeId: RecordRetrievalTypeId, records:
|
|
24
|
+
chunk: (typeId: RecordRetrievalTypeId, records: ParsedRecord[]) => void | Promise<void>;
|
|
25
25
|
options: RetrieveRecordsOptions;
|
|
26
26
|
signal: AbortSignal;
|
|
27
27
|
stream: ReadableStream<Uint8Array>;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
+
import { ParsedRecord, SampleIssue, ValueDelimiterId } from '../../dataView';
|
|
1
2
|
import { ParseTextOptions, ParseTextResult } from '.';
|
|
2
|
-
import { ParsingRecord, SampleIssue, ValueDelimiterId } from '../../dataView';
|
|
3
3
|
export type ParseTextWithDelimiter = (text: string, delimiter: ValueDelimiterId, isComplete: boolean) => Promise<{
|
|
4
4
|
issues: SampleIssue[];
|
|
5
|
-
records:
|
|
5
|
+
records: ParsedRecord[];
|
|
6
6
|
}>;
|
|
7
7
|
/**
|
|
8
8
|
* Reads delimited text for a preview: detects its record delimiter, and which of the candidate value delimiters splits
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { DataFormatId,
|
|
1
|
+
import { DataFormatId, ParsedRecord, RecordDelimiterId, SampleIssue, ValueDelimiterId } from '../../dataView';
|
|
2
2
|
import { RecordRetrievalTypeId, RetrieveRecordsSummary } from '../connector';
|
|
3
3
|
export interface ParserInterface {
|
|
4
|
-
parseStream: (stream: ReadableStream<Uint8Array>, options: ParseStreamOptions, chunk: (typeId: RecordRetrievalTypeId, records:
|
|
4
|
+
parseStream: (stream: ReadableStream<Uint8Array>, options: ParseStreamOptions, chunk: (typeId: RecordRetrievalTypeId, records: ParsedRecord[]) => void | Promise<void>) => Promise<RetrieveRecordsSummary>;
|
|
5
5
|
parseText: (text: string, options: ParseTextOptions) => Promise<ParseTextResult>;
|
|
6
6
|
}
|
|
7
7
|
export interface ParseTextOptions {
|
|
@@ -11,7 +11,7 @@ export interface ParseTextOptions {
|
|
|
11
11
|
}
|
|
12
12
|
export interface ParseTextResult {
|
|
13
13
|
issues: SampleIssue[];
|
|
14
|
-
parsedRecords:
|
|
14
|
+
parsedRecords: ParsedRecord[];
|
|
15
15
|
recordDelimiterId: RecordDelimiterId;
|
|
16
16
|
valueDelimiterId: ValueDelimiterId | undefined;
|
|
17
17
|
}
|