@dpuse/dpuse-shared 1.0.210 → 1.0.211
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -8
- package/dist/dpuse-shared.es.js +233 -208
- package/dist/dpuse-shared.es.js.map +1 -1
- package/dist/types/src/component/module/connector/connectorConfig.schema.d.ts +2 -2
- package/dist/types/src/component/module/connector/index.d.ts +23 -25
- package/dist/types/src/component/module/connector/objectParsing.d.ts +3 -4
- package/dist/types/src/component/module/connector/storeValues.d.ts +4 -0
- package/dist/types/src/component/module/tool/index.d.ts +1 -1
- package/dist/types/src/component/module/tool/parser.d.ts +9 -2
- package/package.json +1 -1
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Defines the configuration metadata for a connector. Used for validation
|
|
3
3
|
* of connector manifests and capability discovery at runtime.
|
|
4
4
|
*/
|
|
5
|
-
export declare const connectorActionNameSchema: import('valibot').UnionSchema<readonly [import('valibot').LiteralSchema<"abortOperation", undefined>, import('valibot').LiteralSchema<"auditObjectContent", undefined>, import('valibot').LiteralSchema<"closeStore", undefined>, import('valibot').LiteralSchema<"createObject", undefined>, import('valibot').LiteralSchema<"describeConnection", undefined>, import('valibot').LiteralSchema<"dropObject", undefined>, import('valibot').LiteralSchema<"findObject", undefined>, import('valibot').LiteralSchema<"getInfo", undefined>, import('valibot').LiteralSchema<"getReadableStream", undefined>, import('valibot').LiteralSchema<"getRecord", undefined>, import('valibot').LiteralSchema<"listNodes", undefined>, import('valibot').LiteralSchema<"loadObjectRecords", undefined>, import('valibot').LiteralSchema<"openStore", undefined>, import('valibot').LiteralSchema<"previewObject", undefined>, import('valibot').LiteralSchema<"queryRecords", undefined>, import('valibot').LiteralSchema<"removeRecords", undefined>, import('valibot').LiteralSchema<"
|
|
5
|
+
export declare const connectorActionNameSchema: import('valibot').UnionSchema<readonly [import('valibot').LiteralSchema<"abortOperation", undefined>, import('valibot').LiteralSchema<"auditObjectContent", undefined>, import('valibot').LiteralSchema<"closeStore", undefined>, import('valibot').LiteralSchema<"createObject", undefined>, import('valibot').LiteralSchema<"describeConnection", undefined>, import('valibot').LiteralSchema<"dropObject", undefined>, import('valibot').LiteralSchema<"findObject", undefined>, import('valibot').LiteralSchema<"getInfo", undefined>, import('valibot').LiteralSchema<"getReadableStream", undefined>, import('valibot').LiteralSchema<"getRecord", undefined>, import('valibot').LiteralSchema<"listNodes", undefined>, import('valibot').LiteralSchema<"loadObjectRecords", undefined>, import('valibot').LiteralSchema<"openStore", undefined>, import('valibot').LiteralSchema<"previewObject", undefined>, import('valibot').LiteralSchema<"queryRecords", undefined>, import('valibot').LiteralSchema<"removeRecords", undefined>, import('valibot').LiteralSchema<"retrieveRecords", undefined>, import('valibot').LiteralSchema<"upsertRecords", undefined>], undefined>;
|
|
6
6
|
export declare const connectorCategoryConfigSchema: import('valibot').StrictObjectSchema<{
|
|
7
7
|
readonly label: import('valibot').StrictObjectSchema<{
|
|
8
8
|
readonly en: import('valibot').StringSchema<undefined>;
|
|
@@ -25,7 +25,7 @@ export declare const connectorUsageConfigSchema: import('valibot').StrictObjectS
|
|
|
25
25
|
}, undefined>;
|
|
26
26
|
export declare const connectorConfigSchema: import('valibot').StrictObjectSchema<{
|
|
27
27
|
readonly typeId: import('valibot').LiteralSchema<"connector", undefined>;
|
|
28
|
-
readonly actionNames: import('valibot').ArraySchema<import('valibot').UnionSchema<readonly [import('valibot').LiteralSchema<"abortOperation", undefined>, import('valibot').LiteralSchema<"auditObjectContent", undefined>, import('valibot').LiteralSchema<"closeStore", undefined>, import('valibot').LiteralSchema<"createObject", undefined>, import('valibot').LiteralSchema<"describeConnection", undefined>, import('valibot').LiteralSchema<"dropObject", undefined>, import('valibot').LiteralSchema<"findObject", undefined>, import('valibot').LiteralSchema<"getInfo", undefined>, import('valibot').LiteralSchema<"getReadableStream", undefined>, import('valibot').LiteralSchema<"getRecord", undefined>, import('valibot').LiteralSchema<"listNodes", undefined>, import('valibot').LiteralSchema<"loadObjectRecords", undefined>, import('valibot').LiteralSchema<"openStore", undefined>, import('valibot').LiteralSchema<"previewObject", undefined>, import('valibot').LiteralSchema<"queryRecords", undefined>, import('valibot').LiteralSchema<"removeRecords", undefined>, import('valibot').LiteralSchema<"
|
|
28
|
+
readonly actionNames: import('valibot').ArraySchema<import('valibot').UnionSchema<readonly [import('valibot').LiteralSchema<"abortOperation", undefined>, import('valibot').LiteralSchema<"auditObjectContent", undefined>, import('valibot').LiteralSchema<"closeStore", undefined>, import('valibot').LiteralSchema<"createObject", undefined>, import('valibot').LiteralSchema<"describeConnection", undefined>, import('valibot').LiteralSchema<"dropObject", undefined>, import('valibot').LiteralSchema<"findObject", undefined>, import('valibot').LiteralSchema<"getInfo", undefined>, import('valibot').LiteralSchema<"getReadableStream", undefined>, import('valibot').LiteralSchema<"getRecord", undefined>, import('valibot').LiteralSchema<"listNodes", undefined>, import('valibot').LiteralSchema<"loadObjectRecords", undefined>, import('valibot').LiteralSchema<"openStore", undefined>, import('valibot').LiteralSchema<"previewObject", undefined>, import('valibot').LiteralSchema<"queryRecords", undefined>, import('valibot').LiteralSchema<"removeRecords", undefined>, import('valibot').LiteralSchema<"retrieveRecords", undefined>, import('valibot').LiteralSchema<"upsertRecords", undefined>], undefined>, undefined>;
|
|
29
29
|
readonly category: import('valibot').NullableSchema<import('valibot').StrictObjectSchema<{
|
|
30
30
|
readonly label: import('valibot').StrictObjectSchema<{
|
|
31
31
|
readonly en: import('valibot').StringSchema<undefined>;
|
|
@@ -24,9 +24,7 @@ export interface ConnectorInterface {
|
|
|
24
24
|
previewObject?(options: PreviewObjectOptions): Promise<PreviewObjectResult>;
|
|
25
25
|
queryRecords?(options: QueryRecordsOptions): Promise<QueryRecordsResult>;
|
|
26
26
|
removeRecords?(options: RemoveRecordsOptions): Promise<void>;
|
|
27
|
-
|
|
28
|
-
retrieveRecords?(options: RetrieveRecordsOptions, chunk: (typeId: RecordRetrievalTypeId, records: Record<string, unknown>[] | ParsedRecord[]) => void | Promise<void>, // A returned promise holds the read back until it settles.
|
|
29
|
-
complete: (result: RetrieveRecordsSummary) => void): Promise<void>;
|
|
27
|
+
retrieveRecords?(options: RetrieveRecordsOptions): AsyncGenerator<RetrieveRecordsChunk, RetrieveRecordsSummary, undefined>;
|
|
30
28
|
upsertRecords?(options: UpsertRecordsOptions): Promise<void>;
|
|
31
29
|
}
|
|
32
30
|
export type ConnectorActionName = InferOutput<typeof connectorActionNameSchema>;
|
|
@@ -53,13 +51,13 @@ export interface AuditObjectContentOptions extends EngineConnectorActionOptions
|
|
|
53
51
|
byteLimit: number | undefined;
|
|
54
52
|
chunkSize: number | undefined;
|
|
55
53
|
dataFormatId: DataFormatId;
|
|
56
|
-
encodingId: string;
|
|
54
|
+
encodingId: string | undefined;
|
|
57
55
|
hasHeaders: boolean | undefined;
|
|
58
56
|
parserId: string | undefined;
|
|
59
57
|
path: string;
|
|
60
58
|
stopSignal: AbortSignal | undefined;
|
|
61
59
|
supportsTransferableStreams: boolean;
|
|
62
|
-
valueDelimiterId: ValueDelimiterId;
|
|
60
|
+
valueDelimiterId: ValueDelimiterId | undefined;
|
|
63
61
|
}
|
|
64
62
|
export interface AuditObjectContentResult {
|
|
65
63
|
contentAuditConfig: ContentAuditConfig;
|
|
@@ -67,8 +65,10 @@ export interface AuditObjectContentResult {
|
|
|
67
65
|
}
|
|
68
66
|
export type CloseStoreOptions = EngineConnectorActionOptions;
|
|
69
67
|
export interface CreateObjectOptions extends EngineConnectorActionOptions {
|
|
68
|
+
columnConfigs: ObjectColumnConfig[];
|
|
69
|
+
indexedColumnIndexes: number[];
|
|
70
|
+
keyColumnIndex: number | undefined;
|
|
70
71
|
path: string;
|
|
71
|
-
structure: string;
|
|
72
72
|
}
|
|
73
73
|
export type DescribeConnectionOptions = EngineConnectorActionOptions;
|
|
74
74
|
interface DescribeConnectionResult {
|
|
@@ -91,15 +91,15 @@ export interface GetInfoResult {
|
|
|
91
91
|
info: Record<string, unknown>;
|
|
92
92
|
}
|
|
93
93
|
export interface GetReadableStreamOptions extends EngineConnectorActionOptions {
|
|
94
|
-
id: string;
|
|
95
94
|
path: string;
|
|
96
95
|
}
|
|
97
96
|
export interface GetRecordOptions extends EngineConnectorActionOptions {
|
|
98
|
-
id:
|
|
97
|
+
id: RecordKey;
|
|
99
98
|
path: string;
|
|
100
99
|
}
|
|
100
|
+
export type RecordKey = number | string;
|
|
101
101
|
export interface GetRecordResult {
|
|
102
|
-
record
|
|
102
|
+
record: StoreRecord;
|
|
103
103
|
}
|
|
104
104
|
export interface ListNodesOptions extends EngineConnectorActionOptions {
|
|
105
105
|
folderPath: string;
|
|
@@ -193,44 +193,42 @@ export interface ClassifyPreviewValuesOptions extends EngineConnectorActionOptio
|
|
|
193
193
|
}
|
|
194
194
|
export type PreviewValueStateId = 'blank' | 'empty' | 'invalid' | 'null' | 'valid';
|
|
195
195
|
export interface RemoveRecordsOptions extends EngineConnectorActionOptions {
|
|
196
|
-
keys:
|
|
197
|
-
path: string;
|
|
198
|
-
}
|
|
199
|
-
export interface RetrieveChunksOptions extends EngineConnectorActionOptions {
|
|
200
|
-
chunkSize: number | undefined;
|
|
201
|
-
encodingId: string;
|
|
196
|
+
keys: RecordKey[];
|
|
202
197
|
path: string;
|
|
203
|
-
valueDelimiterId: ValueDelimiterId;
|
|
204
198
|
}
|
|
205
199
|
export interface RetrieveRecordsOptions extends EngineConnectorActionOptions {
|
|
206
200
|
chunkSize: number | undefined;
|
|
207
201
|
dataFormatId: DataFormatId;
|
|
208
|
-
encodingId: string;
|
|
202
|
+
encodingId: string | undefined;
|
|
209
203
|
parserId: string | undefined;
|
|
210
204
|
path: string;
|
|
211
205
|
range?: RecordRange | undefined;
|
|
212
|
-
valueDelimiterId: ValueDelimiterId;
|
|
206
|
+
valueDelimiterId: ValueDelimiterId | undefined;
|
|
213
207
|
}
|
|
214
208
|
export interface RecordRange {
|
|
215
209
|
limit: number;
|
|
216
210
|
offset: number;
|
|
217
211
|
}
|
|
218
212
|
export interface RetrieveRecordsSummary {
|
|
219
|
-
byteCount: number;
|
|
220
|
-
commentLineCount: number;
|
|
221
|
-
emptyLineCount: number;
|
|
222
|
-
lineCount: number;
|
|
223
|
-
nonUniformRecordCount: number;
|
|
224
213
|
recordCount: number;
|
|
225
214
|
totalCount?: number | undefined;
|
|
226
215
|
}
|
|
227
|
-
export type RecordRetrievalTypeId = '
|
|
216
|
+
export type RecordRetrievalTypeId = 'parsingRecordArray' | 'storeRecordArray';
|
|
217
|
+
export type RetrieveRecordsChunk = ParsedRecordsChunk | {
|
|
218
|
+
records: StoreRecord[];
|
|
219
|
+
typeId: 'storeRecordArray';
|
|
220
|
+
};
|
|
221
|
+
export interface ParsedRecordsChunk {
|
|
222
|
+
records: ParsedRecord[];
|
|
223
|
+
typeId: 'parsingRecordArray';
|
|
224
|
+
}
|
|
228
225
|
export interface UpsertRecordsOptions extends EngineConnectorActionOptions {
|
|
229
|
-
records:
|
|
226
|
+
records: StoreRecord[];
|
|
230
227
|
path: string;
|
|
231
228
|
}
|
|
232
229
|
export type ConnectorUsageId = InferOutput<typeof connectorUsageIdSchema>;
|
|
233
230
|
export { auditObjectContentWithParser, buildPreviewObjectResult, retrieveRecordsWithParser } from './objectParsing';
|
|
231
|
+
export { convertStoreRecordToParsedRecord, convertStoreValueToText } from './storeValues';
|
|
234
232
|
export { readTableSchema } from './tableSchema';
|
|
235
233
|
export declare function determineConnectorUsageId(actionNames: ConnectorActionName[]): ConnectorUsageId;
|
|
236
234
|
export declare const constructConnectorCategoryConfig: (id: string, localeId?: import('../../../locale').LocaleId) => LocalisedConfig<ConnectorCategoryConfig>;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { AuditObjectContentOptions, AuditObjectContentResult, ConnectorUtilities,
|
|
2
|
-
import { DeclaredColumnConfig, DeclaredReadingConfig,
|
|
1
|
+
import { AuditObjectContentOptions, AuditObjectContentResult, ConnectorUtilities, ParsedRecordsChunk, PreviewObjectResult, RetrieveRecordsOptions, RetrieveRecordsSummary } from '.';
|
|
2
|
+
import { DeclaredColumnConfig, DeclaredReadingConfig, ReadingChoiceConfig } from '../../dataView';
|
|
3
3
|
import { FilePreviewResult, ToolConfig } from '../tool';
|
|
4
4
|
export declare function buildPreviewObjectResult(input: {
|
|
5
5
|
asAt: number;
|
|
@@ -12,12 +12,11 @@ export declare function buildPreviewObjectResult(input: {
|
|
|
12
12
|
toolConfigs: ToolConfig[];
|
|
13
13
|
}): Promise<PreviewObjectResult>;
|
|
14
14
|
export declare function retrieveRecordsWithParser(input: {
|
|
15
|
-
chunk: (typeId: RecordRetrievalTypeId, records: ParsedRecord[]) => void | Promise<void>;
|
|
16
15
|
options: RetrieveRecordsOptions;
|
|
17
16
|
signal: AbortSignal;
|
|
18
17
|
stream: ReadableStream<Uint8Array>;
|
|
19
18
|
toolConfigs: ToolConfig[];
|
|
20
|
-
}):
|
|
19
|
+
}): AsyncGenerator<ParsedRecordsChunk, RetrieveRecordsSummary, undefined>;
|
|
21
20
|
export declare function auditObjectContentWithParser(input: {
|
|
22
21
|
constructColumnAuditor: ConnectorUtilities['constructColumnAuditor'];
|
|
23
22
|
declaredColumnConfigs?: DeclaredColumnConfig[] | undefined;
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { ParsedRecord } from '../../dataView';
|
|
2
|
+
import { StoreRecord, StoreValue } from '.';
|
|
3
|
+
export declare function convertStoreRecordToParsedRecord(storeRecord: StoreRecord): ParsedRecord;
|
|
4
|
+
export declare function convertStoreValueToText(storeValue: StoreValue): string | null;
|
|
@@ -5,7 +5,7 @@ export { toolConfigSchema } from './toolConfig.schema';
|
|
|
5
5
|
export type ToolConfig = InferOutput<typeof toolConfigSchema>;
|
|
6
6
|
export type ParserConfig = InferOutput<typeof parserConfigSchema>;
|
|
7
7
|
export type { FilePreviewerInterface, FilePreviewResult } from './filePreviewer';
|
|
8
|
-
export type { ParserInterface, ParseStreamOptions, ParseTextOptions, ParseTextResult } from './parser';
|
|
8
|
+
export type { ParserInterface, ParseStreamOptions, ParseStreamSummary, ParseTextOptions, ParseTextResult } from './parser';
|
|
9
9
|
export { parseDelimitedText, type ParseTextWithDelimiter } from './delimitedText';
|
|
10
10
|
export declare const TOOL_NAME_PREFIX = "dpuse-tool-";
|
|
11
11
|
export declare function listParserToolConfigs(toolConfigs: ToolConfig[], dataFormatId: DataFormatId): ToolConfig[];
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { DataFormatId, ParsedRecord, RecordDelimiterId, SampleIssue, ValueDelimiterId } from '../../dataView';
|
|
2
|
-
import {
|
|
2
|
+
import { ParsedRecordsChunk, RetrieveRecordsSummary } from '../connector';
|
|
3
3
|
export interface ParserInterface {
|
|
4
|
-
parseStream: (stream: ReadableStream<Uint8Array>, options: ParseStreamOptions
|
|
4
|
+
parseStream: (stream: ReadableStream<Uint8Array>, options: ParseStreamOptions) => AsyncGenerator<ParsedRecordsChunk, ParseStreamSummary, undefined>;
|
|
5
5
|
parseText: (text: string, options: ParseTextOptions) => Promise<ParseTextResult>;
|
|
6
6
|
}
|
|
7
7
|
export interface ParseTextOptions {
|
|
@@ -22,3 +22,10 @@ export interface ParseStreamOptions {
|
|
|
22
22
|
encodingId: string;
|
|
23
23
|
valueDelimiterId: ValueDelimiterId;
|
|
24
24
|
}
|
|
25
|
+
export interface ParseStreamSummary extends RetrieveRecordsSummary {
|
|
26
|
+
byteCount: number;
|
|
27
|
+
commentLineCount: number;
|
|
28
|
+
emptyLineCount: number;
|
|
29
|
+
lineCount: number;
|
|
30
|
+
nonUniformRecordCount: number;
|
|
31
|
+
}
|