@dpuse/dpuse-shared 1.0.183 → 1.0.184
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -6
- package/dist/dpuse-shared.es.js +42 -59
- package/dist/dpuse-shared.es.js.map +1 -1
- package/dist/types/src/component/connection/index.d.ts +4 -14
- package/dist/types/src/component/dataView/index.d.ts +24 -34
- package/dist/types/src/component/module/connector/index.d.ts +4 -3
- package/dist/types/src/component/module/connector/objectParsing.d.ts +3 -3
- package/dist/types/src/component/module/tool/delimitedText.d.ts +2 -2
- package/dist/types/src/component/module/tool/parser.d.ts +2 -2
- package/package.json +5 -5
|
@@ -1,5 +1,7 @@
|
|
|
1
|
+
import { BaseConfig } from '../../index';
|
|
1
2
|
import { ConnectorConfig } from '../module/connector';
|
|
2
3
|
import { LocaleLabel } from '../../locale/label';
|
|
4
|
+
import { LocalisedConfig } from '../../locale';
|
|
3
5
|
import { ComponentInstanceConfig } from '..';
|
|
4
6
|
import { DataSubtypeId, DataTypeId } from '../dataView';
|
|
5
7
|
export interface ConnectionConfig extends ComponentInstanceConfig {
|
|
@@ -31,28 +33,16 @@ export interface ConnectionDescriptionConfig {
|
|
|
31
33
|
columns: ObjectColumnConfig[];
|
|
32
34
|
}[];
|
|
33
35
|
}
|
|
34
|
-
export interface ConnectionNodeConfig {
|
|
35
|
-
id: string;
|
|
36
|
-
label: string;
|
|
37
|
-
description: string;
|
|
38
|
-
icon: string | null;
|
|
39
|
-
iconDark: string | null;
|
|
36
|
+
export interface ConnectionNodeConfig extends LocalisedConfig<BaseConfig> {
|
|
40
37
|
childCount: number | undefined;
|
|
41
|
-
childNodes: ConnectionNodeConfig[];
|
|
42
38
|
extension: string | undefined;
|
|
43
39
|
folderPath: string;
|
|
44
|
-
handle: DPAFileSystemFileHandle | undefined;
|
|
45
40
|
lastModifiedAt: number | undefined;
|
|
46
41
|
mimeType: string | undefined;
|
|
47
42
|
name: string;
|
|
48
43
|
size: number | undefined;
|
|
49
|
-
typeId:
|
|
50
|
-
}
|
|
51
|
-
interface DPAFileSystemFileHandle {
|
|
52
|
-
readonly kind: 'file';
|
|
53
|
-
getFile(): Promise<File>;
|
|
44
|
+
typeId: 'folder' | 'object';
|
|
54
45
|
}
|
|
55
|
-
type NodeTypeId = 'folder' | 'object';
|
|
56
46
|
export interface ObjectColumnConfig {
|
|
57
47
|
blankValueCount: number | undefined;
|
|
58
48
|
dataTypeId: DataTypeId;
|
|
@@ -5,63 +5,58 @@ export interface DataViewConfig extends ComponentInstanceConfig {
|
|
|
5
5
|
typeId: 'dataView';
|
|
6
6
|
connectionId: string;
|
|
7
7
|
connectionNodeConfig: ConnectionNodeConfig | undefined;
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
contentAuditConfig:
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
export interface
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
chosenDecimalSeparatorId?: DecimalSeparatorId | undefined;
|
|
23
|
-
chosenNullWordsAreEmpty?: boolean | undefined;
|
|
24
|
-
}
|
|
25
|
-
export interface ReadConfig {
|
|
26
|
-
commentMarkId?: string | undefined;
|
|
27
|
-
commentMarkOtherCharSeq?: string | undefined;
|
|
8
|
+
readingConfig: ReadingConfig | undefined;
|
|
9
|
+
sampleConfig: SampleConfig | undefined;
|
|
10
|
+
contentAuditConfig: ContentAuditConfig | undefined;
|
|
11
|
+
columnChoiceConfigs: ColumnChoiceConfigs | undefined;
|
|
12
|
+
storeConnectorId: string | undefined;
|
|
13
|
+
}
|
|
14
|
+
export type ColumnChoiceConfigs = Partial<Record<string, ColumnChoiceConfig>>;
|
|
15
|
+
export interface ColumnChoiceConfig {
|
|
16
|
+
dataSubtypeId?: DataSubtypeId | undefined;
|
|
17
|
+
dataTypeId?: DataTypeId | undefined;
|
|
18
|
+
decimalSeparatorId?: DecimalSeparatorId | undefined;
|
|
19
|
+
nullWordsAreEmpty?: boolean | undefined;
|
|
20
|
+
}
|
|
21
|
+
export interface ReadingConfig {
|
|
28
22
|
dataFormatId: DataFormatId;
|
|
29
23
|
encodingConfidenceLevel: number | undefined;
|
|
30
24
|
encodingId: string | undefined;
|
|
31
25
|
fileType: FileTypeResult | undefined;
|
|
32
26
|
hasHeaders: boolean | undefined;
|
|
33
|
-
linesToSkipAtStart?: number | undefined;
|
|
34
27
|
parserId?: string | undefined;
|
|
28
|
+
recordDelimiterId: RecordDelimiterId | undefined;
|
|
29
|
+
valueDelimiterId: ValueDelimiterId | undefined;
|
|
30
|
+
commentMarkId?: string | undefined;
|
|
31
|
+
commentMarkOtherCharSeq?: string | undefined;
|
|
32
|
+
linesToSkipAtStart?: number | undefined;
|
|
35
33
|
quoteEscapeChar?: string | undefined;
|
|
36
34
|
quoteMarkId?: string | undefined;
|
|
37
35
|
quoteMarkOtherCharSeq?: string | undefined;
|
|
38
|
-
recordDelimiterId: RecordDelimiterId | undefined;
|
|
39
36
|
recordDelimiterOtherCharSeq?: string | undefined;
|
|
40
37
|
skipEmptyLines?: boolean | undefined;
|
|
41
38
|
skipLinesWithEmptyValues?: boolean | undefined;
|
|
42
39
|
skipLinesWithErrors?: boolean | undefined;
|
|
43
|
-
valueDelimiterId: ValueDelimiterId | undefined;
|
|
44
40
|
valueDelimiterOtherCharSeq?: string | undefined;
|
|
45
41
|
valueTrimMethodId?: ValueTrimMethodId | undefined;
|
|
46
42
|
}
|
|
47
|
-
export interface
|
|
48
|
-
|
|
49
|
-
issues?: PreviewIssue[];
|
|
43
|
+
export interface SampleConfig {
|
|
44
|
+
issues?: SampleIssue[];
|
|
50
45
|
parsedRecords: ParsingRecord[];
|
|
51
46
|
size: number | undefined;
|
|
52
47
|
text: string | undefined;
|
|
53
48
|
}
|
|
54
|
-
export interface
|
|
49
|
+
export interface SampleIssue {
|
|
55
50
|
endLineNumber?: number | undefined;
|
|
56
51
|
fieldNumber?: number | undefined;
|
|
57
52
|
id: string;
|
|
58
53
|
lineNumber: number | undefined;
|
|
59
54
|
message: string;
|
|
60
55
|
recordNumber?: number | undefined;
|
|
61
|
-
severityId:
|
|
56
|
+
severityId: SampleIssueSeverityId;
|
|
62
57
|
value?: string | undefined;
|
|
63
58
|
}
|
|
64
|
-
export type
|
|
59
|
+
export type SampleIssueSeverityId = 'error' | 'info' | 'warning';
|
|
65
60
|
export type DataFormatId = 'arrow' | 'avro' | 'dtv' | 'json' | 'jsonl' | 'markdown' | 'ods' | 'parquet' | 'sav' | 'sqlite' | 'xlsx' | 'xml' | 'unknown';
|
|
66
61
|
export type ParsingRecord = ParsingResult[];
|
|
67
62
|
interface ParsingResult {
|
|
@@ -79,16 +74,11 @@ export interface ContentAuditConfig {
|
|
|
79
74
|
emptyLineCount: number | undefined;
|
|
80
75
|
invalidFieldLengthCount: number | undefined;
|
|
81
76
|
duration: number;
|
|
82
|
-
hasHeaders: boolean;
|
|
83
77
|
isComplete: boolean;
|
|
84
78
|
lineCount: number | undefined;
|
|
85
79
|
recordCount: number;
|
|
86
80
|
sourceWaitDuration: number | undefined;
|
|
87
81
|
}
|
|
88
|
-
export type StoredContentAuditConfig = Omit<ContentAuditConfig, 'columns' | 'hasHeaders'>;
|
|
89
|
-
interface RelationshipsAuditConfig {
|
|
90
|
-
placeholder: string;
|
|
91
|
-
}
|
|
92
82
|
export type DataTypeId = 'boolean' | 'numeric' | 'string' | 'temporal' | 'unknown';
|
|
93
83
|
export type DataSubtypeId = NumericSubtypeId | StringSubtypeId | TemporalSubtypeId;
|
|
94
84
|
export type NumericSubtypeId = 'bigint' | 'integer' | 'decimal';
|
|
@@ -5,7 +5,7 @@ import { LocalisedConfig } from '../../../locale';
|
|
|
5
5
|
import { ToolConfig } from '../tool';
|
|
6
6
|
import { ConnectionConfig, ConnectionDescriptionConfig, ConnectionNodeConfig, ObjectColumnConfig } from '../../connection';
|
|
7
7
|
import { connectorActionNameSchema, connectorCategoryConfigSchema, connectorConfigSchema, connectorItemTypeConfigSchema, connectorItemTypeIdSchema, connectorUsageConfigSchema, connectorUsageIdSchema } from './connectorConfig.schema';
|
|
8
|
-
import { ContentAuditConfig, DataFormatId, InferenceRecord, InferenceSummary, ParsingRecord,
|
|
8
|
+
import { ContentAuditConfig, DataFormatId, InferenceRecord, InferenceSummary, ParsingRecord, ReadingConfig, SampleConfig, ValueDelimiterId } from '../../dataView';
|
|
9
9
|
export { connectorConfigSchema } from './connectorConfig.schema';
|
|
10
10
|
export { auditObjectContentWithParser, buildPreviewObjectResult, type ObjectFilePreview, retrieveRecordsWithParser } from './objectParsing';
|
|
11
11
|
export interface ConnectorInterface extends Component {
|
|
@@ -67,6 +67,7 @@ export interface AuditObjectContentOptions extends EngineConnectorActionOptions
|
|
|
67
67
|
}
|
|
68
68
|
export interface AuditObjectContentResult {
|
|
69
69
|
contentAuditConfig: ContentAuditConfig;
|
|
70
|
+
hasHeaders: boolean;
|
|
70
71
|
}
|
|
71
72
|
export type CloseStoreOptions = EngineConnectorActionOptions;
|
|
72
73
|
export interface CreateObjectOptions extends EngineConnectorActionOptions {
|
|
@@ -152,8 +153,8 @@ export interface PreviewObjectOptions extends EngineConnectorActionOptions {
|
|
|
152
153
|
}
|
|
153
154
|
export interface PreviewObjectResult {
|
|
154
155
|
contentAuditConfig: ContentAuditConfig;
|
|
155
|
-
|
|
156
|
-
|
|
156
|
+
sampleConfig: SampleConfig;
|
|
157
|
+
readingConfig: ReadingConfig;
|
|
157
158
|
}
|
|
158
159
|
export interface QueryRecordsOptions extends EngineConnectorActionOptions {
|
|
159
160
|
aggregates: QueryAggregate[];
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { FileTypeResult } from 'file-type';
|
|
2
|
-
import { AuditObjectContentOptions, ColumnAuditor, ConnectorUtilities, PreviewObjectResult, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
|
|
3
|
-
import {
|
|
2
|
+
import { AuditObjectContentOptions, AuditObjectContentResult, ColumnAuditor, ConnectorUtilities, PreviewObjectResult, RecordRetrievalTypeId, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
|
|
3
|
+
import { DataFormatId, ParsingRecord, ValueDelimiterId } from '../../dataView';
|
|
4
4
|
import { ToolConfig } from '../tool';
|
|
5
5
|
export interface ObjectFilePreview {
|
|
6
6
|
bytes: Uint8Array;
|
|
@@ -34,4 +34,4 @@ export declare function auditObjectContentWithParser(input: {
|
|
|
34
34
|
signal: AbortSignal;
|
|
35
35
|
stream: ReadableStream<Uint8Array>;
|
|
36
36
|
toolConfigs: ToolConfig[];
|
|
37
|
-
}): Promise<
|
|
37
|
+
}): Promise<AuditObjectContentResult>;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { ParseTextOptions, ParseTextResult } from '.';
|
|
2
|
-
import { ParsingRecord,
|
|
2
|
+
import { ParsingRecord, SampleIssue, ValueDelimiterId } from '../../dataView';
|
|
3
3
|
export type ParseTextWithDelimiter = (text: string, delimiter: ValueDelimiterId, isComplete: boolean) => Promise<{
|
|
4
|
-
issues:
|
|
4
|
+
issues: SampleIssue[];
|
|
5
5
|
records: ParsingRecord[];
|
|
6
6
|
}>;
|
|
7
7
|
/**
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { DataFormatId, ParsingRecord,
|
|
1
|
+
import { DataFormatId, ParsingRecord, RecordDelimiterId, SampleIssue, ValueDelimiterId } from '../../dataView';
|
|
2
2
|
import { RecordRetrievalTypeId, RetrieveRecordsSummary } from '../connector';
|
|
3
3
|
export interface ParserInterface {
|
|
4
4
|
parseStream: (stream: ReadableStream<Uint8Array>, options: ParseStreamOptions, chunk: (typeId: RecordRetrievalTypeId, records: ParsingRecord[]) => void | Promise<void>) => Promise<RetrieveRecordsSummary>;
|
|
@@ -10,7 +10,7 @@ export interface ParseTextOptions {
|
|
|
10
10
|
valueDelimiterIds: ValueDelimiterId[];
|
|
11
11
|
}
|
|
12
12
|
export interface ParseTextResult {
|
|
13
|
-
issues:
|
|
13
|
+
issues: SampleIssue[];
|
|
14
14
|
parsedRecords: ParsingRecord[];
|
|
15
15
|
recordDelimiterId: RecordDelimiterId;
|
|
16
16
|
valueDelimiterId: ValueDelimiterId | undefined;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dpuse/dpuse-shared",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.184",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "Common constants, types and utilities used across DPUse projects.",
|
|
6
6
|
"license": "MIT",
|
|
@@ -64,19 +64,19 @@
|
|
|
64
64
|
"valibot": "^1.5.0"
|
|
65
65
|
},
|
|
66
66
|
"devDependencies": {
|
|
67
|
-
"@dpuse/dpuse-development": "^1.0.
|
|
68
|
-
"@dpuse/eslint-config-dpuse": "^1.0.
|
|
67
|
+
"@dpuse/dpuse-development": "^1.0.147",
|
|
68
|
+
"@dpuse/eslint-config-dpuse": "^1.0.149",
|
|
69
69
|
"@types/node": "^26.6.6",
|
|
70
70
|
"@vitest/coverage-v8": "^5.1.0",
|
|
71
71
|
"eslint": "^10.14.0",
|
|
72
|
-
"fallow": "^3.
|
|
72
|
+
"fallow": "^3.40.0",
|
|
73
73
|
"fast-check": "^4.10.2",
|
|
74
74
|
"iconv-lite": "^0.7.3",
|
|
75
75
|
"jschardet": "^4.1.0",
|
|
76
76
|
"prettier": "^3.9.10",
|
|
77
77
|
"sonda": "^0.14.0",
|
|
78
78
|
"typescript": "^6.0.3",
|
|
79
|
-
"vite": "^8.3.
|
|
79
|
+
"vite": "^8.3.6",
|
|
80
80
|
"vite-plugin-dts": "^5.1.2",
|
|
81
81
|
"vitest": "^5.1.0"
|
|
82
82
|
}
|