@dpuse/dpuse-shared 1.0.183 → 1.0.185
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -6
- package/dist/dpuse-shared.es.js +42 -59
- package/dist/dpuse-shared.es.js.map +1 -1
- package/dist/types/src/component/connection/index.d.ts +4 -14
- package/dist/types/src/component/dataView/index.d.ts +30 -41
- package/dist/types/src/component/module/connector/index.d.ts +10 -9
- package/dist/types/src/component/module/connector/objectParsing.d.ts +4 -4
- package/dist/types/src/component/module/tool/delimitedText.d.ts +3 -3
- package/dist/types/src/component/module/tool/parser.d.ts +4 -4
- package/package.json +5 -5
|
@@ -1,5 +1,7 @@
|
|
|
1
|
+
import { BaseConfig } from '../../index';
|
|
1
2
|
import { ConnectorConfig } from '../module/connector';
|
|
2
3
|
import { LocaleLabel } from '../../locale/label';
|
|
4
|
+
import { LocalisedConfig } from '../../locale';
|
|
3
5
|
import { ComponentInstanceConfig } from '..';
|
|
4
6
|
import { DataSubtypeId, DataTypeId } from '../dataView';
|
|
5
7
|
export interface ConnectionConfig extends ComponentInstanceConfig {
|
|
@@ -31,28 +33,16 @@ export interface ConnectionDescriptionConfig {
|
|
|
31
33
|
columns: ObjectColumnConfig[];
|
|
32
34
|
}[];
|
|
33
35
|
}
|
|
34
|
-
export interface ConnectionNodeConfig {
|
|
35
|
-
id: string;
|
|
36
|
-
label: string;
|
|
37
|
-
description: string;
|
|
38
|
-
icon: string | null;
|
|
39
|
-
iconDark: string | null;
|
|
36
|
+
export interface ConnectionNodeConfig extends LocalisedConfig<BaseConfig> {
|
|
40
37
|
childCount: number | undefined;
|
|
41
|
-
childNodes: ConnectionNodeConfig[];
|
|
42
38
|
extension: string | undefined;
|
|
43
39
|
folderPath: string;
|
|
44
|
-
handle: DPAFileSystemFileHandle | undefined;
|
|
45
40
|
lastModifiedAt: number | undefined;
|
|
46
41
|
mimeType: string | undefined;
|
|
47
42
|
name: string;
|
|
48
43
|
size: number | undefined;
|
|
49
|
-
typeId:
|
|
50
|
-
}
|
|
51
|
-
interface DPAFileSystemFileHandle {
|
|
52
|
-
readonly kind: 'file';
|
|
53
|
-
getFile(): Promise<File>;
|
|
44
|
+
typeId: 'folder' | 'object';
|
|
54
45
|
}
|
|
55
|
-
type NodeTypeId = 'folder' | 'object';
|
|
56
46
|
export interface ObjectColumnConfig {
|
|
57
47
|
blankValueCount: number | undefined;
|
|
58
48
|
dataTypeId: DataTypeId;
|
|
@@ -5,72 +5,66 @@ export interface DataViewConfig extends ComponentInstanceConfig {
|
|
|
5
5
|
typeId: 'dataView';
|
|
6
6
|
connectionId: string;
|
|
7
7
|
connectionNodeConfig: ConnectionNodeConfig | undefined;
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
contentAuditConfig:
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
storeConnectorId?: string | undefined;
|
|
8
|
+
readingConfig: ReadingConfig | undefined;
|
|
9
|
+
sampleConfig: SampleConfig | undefined;
|
|
10
|
+
contentAuditConfig: ContentAuditConfig | undefined;
|
|
11
|
+
columnChoiceConfigs: ColumnChoiceConfigs | undefined;
|
|
12
|
+
storeConnectorId: string | undefined;
|
|
14
13
|
}
|
|
15
|
-
export interface
|
|
16
|
-
asAt: number;
|
|
17
|
-
columnConfigs: DataViewColumnConfig[];
|
|
18
|
-
}
|
|
19
|
-
export interface DataViewColumnConfig extends ObjectColumnConfig {
|
|
20
|
-
chosenDataSubtypeId?: DataSubtypeId | undefined;
|
|
21
|
-
chosenDataTypeId?: DataTypeId | undefined;
|
|
22
|
-
chosenDecimalSeparatorId?: DecimalSeparatorId | undefined;
|
|
23
|
-
chosenNullWordsAreEmpty?: boolean | undefined;
|
|
24
|
-
}
|
|
25
|
-
export interface ReadConfig {
|
|
26
|
-
commentMarkId?: string | undefined;
|
|
27
|
-
commentMarkOtherCharSeq?: string | undefined;
|
|
14
|
+
export interface ReadingConfig {
|
|
28
15
|
dataFormatId: DataFormatId;
|
|
29
16
|
encodingConfidenceLevel: number | undefined;
|
|
30
17
|
encodingId: string | undefined;
|
|
31
18
|
fileType: FileTypeResult | undefined;
|
|
32
19
|
hasHeaders: boolean | undefined;
|
|
33
|
-
linesToSkipAtStart?: number | undefined;
|
|
34
20
|
parserId?: string | undefined;
|
|
21
|
+
recordDelimiterId: RecordDelimiterId | undefined;
|
|
22
|
+
valueDelimiterId: ValueDelimiterId | undefined;
|
|
23
|
+
commentMarkId?: string | undefined;
|
|
24
|
+
commentMarkOtherCharSeq?: string | undefined;
|
|
25
|
+
linesToSkipAtStart?: number | undefined;
|
|
35
26
|
quoteEscapeChar?: string | undefined;
|
|
36
27
|
quoteMarkId?: string | undefined;
|
|
37
28
|
quoteMarkOtherCharSeq?: string | undefined;
|
|
38
|
-
recordDelimiterId: RecordDelimiterId | undefined;
|
|
39
29
|
recordDelimiterOtherCharSeq?: string | undefined;
|
|
40
30
|
skipEmptyLines?: boolean | undefined;
|
|
41
31
|
skipLinesWithEmptyValues?: boolean | undefined;
|
|
42
32
|
skipLinesWithErrors?: boolean | undefined;
|
|
43
|
-
valueDelimiterId: ValueDelimiterId | undefined;
|
|
44
33
|
valueDelimiterOtherCharSeq?: string | undefined;
|
|
45
|
-
valueTrimMethodId?:
|
|
34
|
+
valueTrimMethodId?: 'both' | 'left' | 'right' | 'none' | undefined;
|
|
46
35
|
}
|
|
47
|
-
export
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
36
|
+
export type DataFormatId = 'arrow' | 'avro' | 'dtv' | 'json' | 'jsonl' | 'markdown' | 'ods' | 'parquet' | 'sav' | 'sqlite' | 'xlsx' | 'xml' | 'unknown';
|
|
37
|
+
export type RecordDelimiterId = '\n' | '\r' | '\r\n';
|
|
38
|
+
export type ValueDelimiterId = '' | ':' | ',' | '!' | '0x1E' | ';' | ' ' | '\t' | '_' | '0x1F' | '|';
|
|
39
|
+
export type ColumnChoiceConfigs = Partial<Record<string, ColumnChoiceConfig>>;
|
|
40
|
+
export interface ColumnChoiceConfig {
|
|
41
|
+
dataSubtypeId?: DataSubtypeId | undefined;
|
|
42
|
+
dataTypeId?: DataTypeId | undefined;
|
|
43
|
+
decimalSeparatorId?: DecimalSeparatorId | undefined;
|
|
44
|
+
nullWordsAreEmpty?: boolean | undefined;
|
|
45
|
+
}
|
|
46
|
+
export interface SampleConfig {
|
|
47
|
+
issues?: SampleIssue[];
|
|
48
|
+
parsedRecords: ParsedRecord[];
|
|
51
49
|
size: number | undefined;
|
|
52
50
|
text: string | undefined;
|
|
53
51
|
}
|
|
54
|
-
export interface
|
|
52
|
+
export interface SampleIssue {
|
|
55
53
|
endLineNumber?: number | undefined;
|
|
56
54
|
fieldNumber?: number | undefined;
|
|
57
55
|
id: string;
|
|
58
56
|
lineNumber: number | undefined;
|
|
59
57
|
message: string;
|
|
60
58
|
recordNumber?: number | undefined;
|
|
61
|
-
severityId:
|
|
59
|
+
severityId: SampleIssueSeverityId;
|
|
62
60
|
value?: string | undefined;
|
|
63
61
|
}
|
|
64
|
-
export type
|
|
65
|
-
export type
|
|
66
|
-
|
|
67
|
-
interface ParsingResult {
|
|
62
|
+
export type SampleIssueSeverityId = 'error' | 'info' | 'warning';
|
|
63
|
+
export type ParsedRecord = ParsedValue[];
|
|
64
|
+
interface ParsedValue {
|
|
68
65
|
value: string | null;
|
|
69
66
|
valueWasQuoted: boolean;
|
|
70
67
|
}
|
|
71
|
-
export type RecordDelimiterId = '\n' | '\r' | '\r\n';
|
|
72
|
-
export type ValueDelimiterId = '' | ':' | ',' | '!' | '0x1E' | ';' | ' ' | '\t' | '_' | '0x1F' | '|';
|
|
73
|
-
type ValueTrimMethodId = 'both' | 'left' | 'right' | 'none';
|
|
74
68
|
export interface ContentAuditConfig {
|
|
75
69
|
asAt: number;
|
|
76
70
|
byteCount: number;
|
|
@@ -79,16 +73,11 @@ export interface ContentAuditConfig {
|
|
|
79
73
|
emptyLineCount: number | undefined;
|
|
80
74
|
invalidFieldLengthCount: number | undefined;
|
|
81
75
|
duration: number;
|
|
82
|
-
hasHeaders: boolean;
|
|
83
76
|
isComplete: boolean;
|
|
84
77
|
lineCount: number | undefined;
|
|
85
78
|
recordCount: number;
|
|
86
79
|
sourceWaitDuration: number | undefined;
|
|
87
80
|
}
|
|
88
|
-
export type StoredContentAuditConfig = Omit<ContentAuditConfig, 'columns' | 'hasHeaders'>;
|
|
89
|
-
interface RelationshipsAuditConfig {
|
|
90
|
-
placeholder: string;
|
|
91
|
-
}
|
|
92
81
|
export type DataTypeId = 'boolean' | 'numeric' | 'string' | 'temporal' | 'unknown';
|
|
93
82
|
export type DataSubtypeId = NumericSubtypeId | StringSubtypeId | TemporalSubtypeId;
|
|
94
83
|
export type NumericSubtypeId = 'bigint' | 'integer' | 'decimal';
|
|
@@ -5,7 +5,7 @@ import { LocalisedConfig } from '../../../locale';
|
|
|
5
5
|
import { ToolConfig } from '../tool';
|
|
6
6
|
import { ConnectionConfig, ConnectionDescriptionConfig, ConnectionNodeConfig, ObjectColumnConfig } from '../../connection';
|
|
7
7
|
import { connectorActionNameSchema, connectorCategoryConfigSchema, connectorConfigSchema, connectorItemTypeConfigSchema, connectorItemTypeIdSchema, connectorUsageConfigSchema, connectorUsageIdSchema } from './connectorConfig.schema';
|
|
8
|
-
import { ContentAuditConfig, DataFormatId, InferenceRecord, InferenceSummary,
|
|
8
|
+
import { ContentAuditConfig, DataFormatId, InferenceRecord, InferenceSummary, ParsedRecord, ReadingConfig, SampleConfig, ValueDelimiterId } from '../../dataView';
|
|
9
9
|
export { connectorConfigSchema } from './connectorConfig.schema';
|
|
10
10
|
export { auditObjectContentWithParser, buildPreviewObjectResult, type ObjectFilePreview, retrieveRecordsWithParser } from './objectParsing';
|
|
11
11
|
export interface ConnectorInterface extends Component {
|
|
@@ -29,7 +29,7 @@ export interface ConnectorInterface extends Component {
|
|
|
29
29
|
queryRecords?(options: QueryRecordsOptions): Promise<QueryRecordsResult>;
|
|
30
30
|
removeRecords?(options: RemoveRecordsOptions): Promise<void>;
|
|
31
31
|
retrieveChunks?(options: RetrieveChunksOptions, chunk: (data: Uint8Array) => void, complete: () => void): Promise<void>;
|
|
32
|
-
retrieveRecords?(options: RetrieveRecordsOptions, chunk: (typeId: RecordRetrievalTypeId, records: Record<string, unknown>[] |
|
|
32
|
+
retrieveRecords?(options: RetrieveRecordsOptions, chunk: (typeId: RecordRetrievalTypeId, records: Record<string, unknown>[] | ParsedRecord[]) => void | Promise<void>, // A returned promise holds the read back until it settles.
|
|
33
33
|
complete: (result: RetrieveRecordsSummary) => void): Promise<void>;
|
|
34
34
|
upsertRecords?(options: UpsertRecordsOptions): Promise<void>;
|
|
35
35
|
}
|
|
@@ -38,11 +38,11 @@ export type ConnectorConstructor = new (connectorUtilities: ConnectorUtilities,
|
|
|
38
38
|
export interface ConnectorUtilities {
|
|
39
39
|
constructColumnAuditor: (hasHeaders: boolean | undefined) => ColumnAuditor;
|
|
40
40
|
hasReadableStreamTransferSupport(): boolean;
|
|
41
|
-
inferValues: (parsedRecord:
|
|
42
|
-
inferDataTypes: (parsedRecords:
|
|
41
|
+
inferValues: (parsedRecord: ParsedRecord, columnConfigs: ObjectColumnConfig[], hasLeadingRecord: boolean) => InferenceRecord;
|
|
42
|
+
inferDataTypes: (parsedRecords: ParsedRecord[], hasHeaders?: boolean) => InferenceSummary;
|
|
43
43
|
}
|
|
44
44
|
export interface ColumnAuditor {
|
|
45
|
-
auditRecords: (parsedRecords:
|
|
45
|
+
auditRecords: (parsedRecords: ParsedRecord[]) => void;
|
|
46
46
|
completeAudit: () => {
|
|
47
47
|
columnConfigs: ObjectColumnConfig[];
|
|
48
48
|
hasHeaders: boolean;
|
|
@@ -67,6 +67,7 @@ export interface AuditObjectContentOptions extends EngineConnectorActionOptions
|
|
|
67
67
|
}
|
|
68
68
|
export interface AuditObjectContentResult {
|
|
69
69
|
contentAuditConfig: ContentAuditConfig;
|
|
70
|
+
hasHeaders: boolean;
|
|
70
71
|
}
|
|
71
72
|
export type CloseStoreOptions = EngineConnectorActionOptions;
|
|
72
73
|
export interface CreateObjectOptions extends EngineConnectorActionOptions {
|
|
@@ -152,8 +153,8 @@ export interface PreviewObjectOptions extends EngineConnectorActionOptions {
|
|
|
152
153
|
}
|
|
153
154
|
export interface PreviewObjectResult {
|
|
154
155
|
contentAuditConfig: ContentAuditConfig;
|
|
155
|
-
|
|
156
|
-
|
|
156
|
+
sampleConfig: SampleConfig;
|
|
157
|
+
readingConfig: ReadingConfig;
|
|
157
158
|
}
|
|
158
159
|
export interface QueryRecordsOptions extends EngineConnectorActionOptions {
|
|
159
160
|
aggregates: QueryAggregate[];
|
|
@@ -185,11 +186,11 @@ export interface QueryRecordsResult {
|
|
|
185
186
|
}
|
|
186
187
|
export interface RecountPreviewColumnsOptions extends EngineConnectorActionOptions {
|
|
187
188
|
hasHeaders: boolean;
|
|
188
|
-
parsedRecords:
|
|
189
|
+
parsedRecords: ParsedRecord[];
|
|
189
190
|
}
|
|
190
191
|
export interface ClassifyPreviewValuesOptions extends EngineConnectorActionOptions {
|
|
191
192
|
columnConfigs: ObjectColumnConfig[];
|
|
192
|
-
parsedRecords:
|
|
193
|
+
parsedRecords: ParsedRecord[];
|
|
193
194
|
}
|
|
194
195
|
export type PreviewValueStateId = 'blank' | 'invalid' | 'missing' | 'valid';
|
|
195
196
|
export interface RemoveRecordsOptions extends EngineConnectorActionOptions {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { FileTypeResult } from 'file-type';
|
|
2
|
-
import { AuditObjectContentOptions, ColumnAuditor, ConnectorUtilities, PreviewObjectResult, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
|
|
3
|
-
import {
|
|
2
|
+
import { AuditObjectContentOptions, AuditObjectContentResult, ColumnAuditor, ConnectorUtilities, PreviewObjectResult, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
|
|
3
|
+
import { DataFormatId, ParsedRecord, ValueDelimiterId } from '../../dataView';
|
|
4
4
|
import { ToolConfig } from '../tool';
|
|
5
5
|
export interface ObjectFilePreview {
|
|
6
6
|
bytes: Uint8Array;
|
|
@@ -21,7 +21,7 @@ export declare function buildPreviewObjectResult(input: {
|
|
|
21
21
|
valueDelimiterId: ValueDelimiterId | undefined;
|
|
22
22
|
}): Promise<PreviewObjectResult>;
|
|
23
23
|
export declare function retrieveRecordsWithParser(input: {
|
|
24
|
-
chunk: (typeId: RecordRetrievalTypeId, records:
|
|
24
|
+
chunk: (typeId: RecordRetrievalTypeId, records: ParsedRecord[]) => void | Promise<void>;
|
|
25
25
|
options: RetrieveRecordsOptions;
|
|
26
26
|
signal: AbortSignal;
|
|
27
27
|
stream: ReadableStream<Uint8Array>;
|
|
@@ -34,4 +34,4 @@ export declare function auditObjectContentWithParser(input: {
|
|
|
34
34
|
signal: AbortSignal;
|
|
35
35
|
stream: ReadableStream<Uint8Array>;
|
|
36
36
|
toolConfigs: ToolConfig[];
|
|
37
|
-
}): Promise<
|
|
37
|
+
}): Promise<AuditObjectContentResult>;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
+
import { ParsedRecord, SampleIssue, ValueDelimiterId } from '../../dataView';
|
|
1
2
|
import { ParseTextOptions, ParseTextResult } from '.';
|
|
2
|
-
import { ParsingRecord, PreviewIssue, ValueDelimiterId } from '../../dataView';
|
|
3
3
|
export type ParseTextWithDelimiter = (text: string, delimiter: ValueDelimiterId, isComplete: boolean) => Promise<{
|
|
4
|
-
issues:
|
|
5
|
-
records:
|
|
4
|
+
issues: SampleIssue[];
|
|
5
|
+
records: ParsedRecord[];
|
|
6
6
|
}>;
|
|
7
7
|
/**
|
|
8
8
|
* Reads delimited text for a preview: detects its record delimiter, and which of the candidate value delimiters splits
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { DataFormatId,
|
|
1
|
+
import { DataFormatId, ParsedRecord, RecordDelimiterId, SampleIssue, ValueDelimiterId } from '../../dataView';
|
|
2
2
|
import { RecordRetrievalTypeId, RetrieveRecordsSummary } from '../connector';
|
|
3
3
|
export interface ParserInterface {
|
|
4
|
-
parseStream: (stream: ReadableStream<Uint8Array>, options: ParseStreamOptions, chunk: (typeId: RecordRetrievalTypeId, records:
|
|
4
|
+
parseStream: (stream: ReadableStream<Uint8Array>, options: ParseStreamOptions, chunk: (typeId: RecordRetrievalTypeId, records: ParsedRecord[]) => void | Promise<void>) => Promise<RetrieveRecordsSummary>;
|
|
5
5
|
parseText: (text: string, options: ParseTextOptions) => Promise<ParseTextResult>;
|
|
6
6
|
}
|
|
7
7
|
export interface ParseTextOptions {
|
|
@@ -10,8 +10,8 @@ export interface ParseTextOptions {
|
|
|
10
10
|
valueDelimiterIds: ValueDelimiterId[];
|
|
11
11
|
}
|
|
12
12
|
export interface ParseTextResult {
|
|
13
|
-
issues:
|
|
14
|
-
parsedRecords:
|
|
13
|
+
issues: SampleIssue[];
|
|
14
|
+
parsedRecords: ParsedRecord[];
|
|
15
15
|
recordDelimiterId: RecordDelimiterId;
|
|
16
16
|
valueDelimiterId: ValueDelimiterId | undefined;
|
|
17
17
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dpuse/dpuse-shared",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.185",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "Common constants, types and utilities used across DPUse projects.",
|
|
6
6
|
"license": "MIT",
|
|
@@ -64,19 +64,19 @@
|
|
|
64
64
|
"valibot": "^1.5.0"
|
|
65
65
|
},
|
|
66
66
|
"devDependencies": {
|
|
67
|
-
"@dpuse/dpuse-development": "^1.0.
|
|
68
|
-
"@dpuse/eslint-config-dpuse": "^1.0.
|
|
67
|
+
"@dpuse/dpuse-development": "^1.0.147",
|
|
68
|
+
"@dpuse/eslint-config-dpuse": "^1.0.149",
|
|
69
69
|
"@types/node": "^26.6.6",
|
|
70
70
|
"@vitest/coverage-v8": "^5.1.0",
|
|
71
71
|
"eslint": "^10.14.0",
|
|
72
|
-
"fallow": "^3.
|
|
72
|
+
"fallow": "^3.40.0",
|
|
73
73
|
"fast-check": "^4.10.2",
|
|
74
74
|
"iconv-lite": "^0.7.3",
|
|
75
75
|
"jschardet": "^4.1.0",
|
|
76
76
|
"prettier": "^3.9.10",
|
|
77
77
|
"sonda": "^0.14.0",
|
|
78
78
|
"typescript": "^6.0.3",
|
|
79
|
-
"vite": "^8.3.
|
|
79
|
+
"vite": "^8.3.6",
|
|
80
80
|
"vite-plugin-dts": "^5.1.2",
|
|
81
81
|
"vitest": "^5.1.0"
|
|
82
82
|
}
|